From 42638afc7568c161f467a4f6f385d597a6e329d9 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 26 Jan 2018 11:47:25 +0200 Subject: [PATCH 01/56] Drop support for EOL Python 2.6 and 3.3 * Drop support for EOL Python 2.6 * Use more helpful assertIn/NotIn instead of assertTrue/False * Drop support for EOL Python 3.3 * Remove redundant Python 3.3 code * Restore code for RHEL 6 and virtualenv for Py2.7 * Revert pipstrap.py to upstream * Merge py26_packages and non_py26_packages into all_packages * Revert changes to *-auto in root * Update by calling letsencrypt-auto-source/build.py * Revert permissions for pipstrap.py --- .travis.yml | 8 ---- acme/acme/__init__.py | 10 ---- acme/acme/crypto_util.py | 4 +- acme/acme/crypto_util_test.py | 12 ++--- acme/setup.py | 9 ---- certbot-apache/setup.py | 2 - certbot-compatibility-test/setup.py | 2 - certbot-dns-cloudflare/setup.py | 2 - certbot-dns-cloudxns/setup.py | 1 - certbot-dns-digitalocean/setup.py | 2 - certbot-dns-dnsimple/setup.py | 1 - certbot-dns-dnsmadeeasy/setup.py | 1 - certbot-dns-google/setup.py | 2 - certbot-dns-luadns/setup.py | 1 - certbot-dns-nsone/setup.py | 1 - certbot-dns-rfc2136/setup.py | 2 - certbot-dns-route53/setup.py | 2 - certbot-nginx/setup.py | 2 - certbot/log.py | 48 ++++--------------- certbot/main.py | 12 ----- docs/contributing.rst | 2 +- docs/install.rst | 2 +- letsencrypt-auto-source/letsencrypt-auto | 35 ++++---------- .../letsencrypt-auto.template | 35 ++++---------- letsencrypt-auto-source/tests/auto_test.py | 16 +++---- letshelp-certbot/letshelp_certbot/apache.py | 4 +- letshelp-certbot/setup.py | 2 - setup.py | 9 ---- tests/letstest/scripts/test_tox.sh | 8 +--- tests/run_http_server.py | 2 +- tools/install_and_test.sh | 4 -- tox.ini | 41 ++++------------ 32 files changed, 56 insertions(+), 228 deletions(-) diff --git a/.travis.yml b/.travis.yml index 35666d8e6..1077d99d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,18 +25,10 @@ matrix: addons: - python: "2.7" env: TOXENV=lint - - python: "2.6" - env: TOXENV=py26 - sudo: required - services: docker - python: "2.7" env: TOXENV=py27-oldest sudo: required services: docker - - python: "3.3" - env: TOXENV=py33 - sudo: required - services: docker - python: "3.6" env: TOXENV=py36 sudo: required diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index 5850fa955..e8a0b16a8 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -10,13 +10,3 @@ supported version: `draft-ietf-acme-01`_. https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ -import sys -import warnings - -for (major, minor) in [(2, 6), (3, 3)]: - if sys.version_info[:2] == (major, minor): - warnings.warn( - "Python {0}.{1} support will be dropped in the next release of " - "acme. Please upgrade your Python version.".format(major, minor), - DeprecationWarning, - ) #pragma: no cover diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index b8fba0348..78ba41d0f 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -5,7 +5,6 @@ import logging import os import re import socket -import sys import OpenSSL @@ -130,8 +129,7 @@ def probe_sni(name, host, port=443, timeout=300, context = OpenSSL.SSL.Context(method) context.set_timeout(timeout) - socket_kwargs = {} if sys.version_info < (2, 7) else { - 'source_address': source_address} + socket_kwargs = {'source_address': source_address} host_protocol_agnostic = None if host == '::' or host == '0' else host diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf..22a507811 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -170,9 +170,9 @@ class MakeCSRTest(unittest.TestCase): self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 1) self.assertEquals(csr.get_extensions()[0].get_data(), @@ -188,9 +188,9 @@ class MakeCSRTest(unittest.TestCase): csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 2) # NOTE: Ideally we would filter by the TLS Feature OID, but diff --git a/acme/setup.py b/acme/setup.py index ce426cf74..ba5c8e6fb 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -25,13 +25,6 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ 'pytest', 'pytest-xdist', @@ -58,10 +51,8 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 38f41e9f1..d7c223a0a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 8f9f897cf..7e1b059e2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -40,10 +40,8 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 612e7259f..d619f1872 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -39,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3157400c6..5d14f3e29 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1a68400fa..ce8fedd46 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 35de47308..06af16759 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a946d00a4..7c0f3ed86 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8585fc848..de881ad84 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -44,10 +44,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4fec37e29..0d580b7ee 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index dca9ebf27..c0ba11470 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bfa72b50b..5161e7a94 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -39,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8df687972..09f8a7d52 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -32,10 +32,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 152f77de8..37c477ef6 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot/log.py b/certbot/log.py index f7c7b126c..e0d2e8f11 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -165,12 +165,7 @@ class ColoredStreamHandler(logging.StreamHandler): """ def __init__(self, stream=None): - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(ColoredStreamHandler, self).__init__(stream) + super(ColoredStreamHandler, self).__init__(stream) self.colored = (sys.stderr.isatty() if stream is None else stream.isatty()) self.red_level = logging.WARNING @@ -184,9 +179,7 @@ class ColoredStreamHandler(logging.StreamHandler): :rtype: str """ - out = (logging.StreamHandler.format(self, record) - if sys.version_info < (2, 7) - else super(ColoredStreamHandler, self).format(record)) + out = super(ColoredStreamHandler, self).format(record) if self.colored and record.levelno >= self.red_level: return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) else: @@ -203,23 +196,14 @@ class MemoryHandler(logging.handlers.MemoryHandler): def __init__(self, target=None): # capacity doesn't matter because should_flush() is overridden capacity = float('inf') - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.__init__( - self, capacity, target=target) - else: - super(MemoryHandler, self).__init__(capacity, target=target) + super(MemoryHandler, self).__init__(capacity, target=target) def close(self): """Close the memory handler, but don't set the target to None.""" # This allows the logging module which may only have a weak # reference to the target handler to properly flush and close it. target = self.target - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.close(self) - else: - super(MemoryHandler, self).close() + super(MemoryHandler, self).close() self.target = target def flush(self, force=False): # pylint: disable=arguments-differ @@ -233,10 +217,7 @@ class MemoryHandler(logging.handlers.MemoryHandler): # This method allows flush() calls in logging.shutdown to be a # noop so we can control when this handler is flushed. if force: - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.flush(self) - else: - super(MemoryHandler, self).flush() + super(MemoryHandler, self).flush() def shouldFlush(self, record): """Should the buffer be automatically flushed? @@ -262,12 +243,7 @@ class TempHandler(logging.StreamHandler): """ def __init__(self): stream = tempfile.NamedTemporaryFile('w', delete=False) - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(TempHandler, self).__init__(stream) + super(TempHandler, self).__init__(stream) self.path = stream.name self._delete = True @@ -278,12 +254,7 @@ class TempHandler(logging.StreamHandler): """ self._delete = False - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.emit(self, record) - else: - super(TempHandler, self).emit(record) + super(TempHandler, self).emit(record) def close(self): """Close the handler and the temporary log file. @@ -299,10 +270,7 @@ class TempHandler(logging.StreamHandler): if self._delete: os.remove(self.path) self._delete = False - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.close(self) - else: - super(TempHandler, self).close() + super(TempHandler, self).close() finally: self.release() diff --git a/certbot/main.py b/certbot/main.py index 32dd69256..76a90d499 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -4,7 +4,6 @@ import functools import logging.handlers import os import sys -import warnings import configobj import josepy as jose @@ -1218,17 +1217,6 @@ def main(cli_args=sys.argv[1:]): # Let plugins_cmd be run as un-privileged user. if config.func != plugins_cmd: raise - deprecation_fmt = ( - "Python %s.%s support will be dropped in the next " - "release of Certbot - please upgrade your Python version.") - # We use the warnings system for Python 2.6 and logging for Python 3 - # because DeprecationWarnings are only reported by default in Python <= 2.6 - # and warnings can be disabled by the user. - if sys.version_info[:2] == (2, 6): - warning = deprecation_fmt % sys.version_info[:2] - warnings.warn(warning, DeprecationWarning) - elif sys.version_info[:2] == (3, 3): - logger.warning(deprecation_fmt, *sys.version_info[:2]) set_displayer(config) diff --git a/docs/contributing.rst b/docs/contributing.rst index 83b607e15..654528e3d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -422,7 +422,7 @@ OS-level dependencies can be installed like so: In general... * ``sudo`` is required as a suggested way of running privileged process -* `Python`_ 2.6/2.7 is required +* `Python`_ 2.7 is required * `Augeas`_ is required for the Python bindings * ``virtualenv`` and ``pip`` are used for managing other python library dependencies diff --git a/docs/install.rst b/docs/install.rst index c18c3cdbc..aec885b62 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,7 +22,7 @@ your system. System Requirements =================== -Certbot currently requires Python 2.6, 2.7, or 3.3+. By default, it requires +Certbot currently requires Python 2.7, or 3.4+. By default, it requires root access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and modify webserver diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8ff7944b5..aed15a8ef 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -246,7 +246,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -781,20 +781,11 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -965,18 +956,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2ce337002..b3d6ab740 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -246,7 +246,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -320,20 +320,11 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -504,18 +495,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index d187452a1..8c2bfc079 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -287,8 +287,8 @@ class AutoTests(TestCase): self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: - self.assertTrue('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This # conveniently sets us up to test the next 2 cases. @@ -296,8 +296,8 @@ class AutoTests(TestCase): # Test when neither phase-1 upgrade nor phase-2 upgrade is # needed (probably a common case): out, err = run_letsencrypt_auto() - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertFalse('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertNotIn('Creating virtual environment...', out) def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" @@ -312,8 +312,8 @@ class AutoTests(TestCase): # Create venv saving the correct bootstrap script version out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: bootstrap_version = f.read() @@ -329,8 +329,8 @@ class AutoTests(TestCase): out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) def test_openssl_failure(self): """Make sure we stop if the openssl signature check fails.""" diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index b13057ca5..f77a6a1b0 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -5,7 +5,6 @@ from __future__ import print_function import argparse import atexit -import contextlib import os import re import shutil @@ -302,8 +301,7 @@ def main(): make_and_verify_selection(args.server_root, tempdir) tarpath = os.path.join(tempdir, "config.tar.gz") - # contextlib.closing used for py26 support - with contextlib.closing(tarfile.open(tarpath, mode="w:gz")) as tar: + with tarfile.open(tarpath, mode="w:gz") as tar: tar.add(tempdir, arcname=".") # TODO: Submit tarpath diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 3ce442b3e..7c8c39068 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -31,10 +31,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/setup.py b/setup.py index ce505a62e..e3824a7f7 100644 --- a/setup.py +++ b/setup.py @@ -52,13 +52,6 @@ install_requires = [ 'zope.interface', ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', @@ -98,10 +91,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index 4c2eb429e..84e4bcd22 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -15,10 +15,4 @@ VENV_BIN=${VENV_PATH}/bin cd letsencrypt ./tools/venv.sh -PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - -if [ $PYVER -eq 26 ] ; then - venv/bin/tox -e py26 -else - venv/bin/tox -e py27 -fi +venv/bin/tox -e py27 diff --git a/tests/run_http_server.py b/tests/run_http_server.py index fd1163816..0e4f8ac79 100644 --- a/tests/run_http_server.py +++ b/tests/run_http_server.py @@ -3,7 +3,7 @@ import sys # Run Python's built-in HTTP server # Usage: python ./tests/run_http_server.py port_num -# NOTE: This script should be compatible with 2.6, 2.7, 3.3+ +# NOTE: This script should be compatible with 2.7, 3.4+ # sys.argv (port number) is passed as-is to the HTTP server module runpy.run_module( diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 0d39e0594..f0385470b 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -18,10 +18,6 @@ for requirement in "$@" ; do pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] if [ $pkg = "." ]; then pkg="certbot" - else - # Work around a bug in pytest/importlib for the deprecated Python 3.3. - # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. - pkg=$(echo "$pkg" | tr - _) fi "$(dirname $0)/pytest.sh" --pyargs $pkg done diff --git a/tox.ini b/tox.ini index 20f5cda32..971aa7631 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py{26,33,34,35,36},cover,lint +envlist = modification,py{34,35,36},cover,lint [base] # pip installs the requested packages in editable mode @@ -14,25 +14,22 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -py26_packages = +all_packages = acme[dev] \ .[dev] \ certbot-apache \ certbot-dns-cloudflare \ + certbot-dns-cloudxns \ certbot-dns-digitalocean \ + certbot-dns-dnsimple \ + certbot-dns-dnsmadeeasy \ certbot-dns-google \ + certbot-dns-luadns \ + certbot-dns-nsone \ certbot-dns-rfc2136 \ certbot-dns-route53 \ certbot-nginx \ letshelp-certbot -non_py26_packages = - certbot-dns-cloudxns \ - certbot-dns-dnsimple \ - certbot-dns-dnsmadeeasy \ - certbot-dns-luadns \ - certbot-dns-nsone -all_packages = - {[base]py26_packages} {[base]non_py26_packages} install_packages = {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} source_paths = @@ -54,32 +51,15 @@ source_paths = letshelp-certbot/letshelp_certbot tests/lock_test.py -[testenv:py26] -commands = - {[base]install_and_test} {[base]py26_packages} - python tests/lock_test.py -deps = - setuptools==36.8.0 - wheel==0.29.0 -passenv = TRAVIS - [testenv] commands = - {[testenv:py26]commands} - {[base]install_and_test} {[base]non_py26_packages} + {[base]install_and_test} {[base]all_packages} + python tests/lock_test.py setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 passenv = - {[testenv:py26]passenv} - -[testenv:py33] -commands = - {[testenv]commands} -deps = - wheel==0.29.0 -passenv = - {[testenv]passenv} + TRAVIS [testenv:py27-oldest] commands = @@ -104,7 +84,6 @@ passenv = {[testenv]passenv} [testenv:lint] -# recent versions of pylint do not support Python 2.6 (#97, #187) basepython = python2.7 # separating into multiple invocations disables cross package # duplicate code checking; if one of the commands fails, others will From 73bd801f352fb9bad7fe8bc35c368f573c15a21e Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:21:02 -0800 Subject: [PATCH 02/56] add and use request_authorizations --- acme/acme/client.py | 14 ++++++++++++++ certbot/auth_handler.py | 10 ++++++---- certbot/client.py | 23 ++++++++--------------- certbot/main.py | 2 +- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 1f4ae4fad..6b4d65233 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -671,6 +671,7 @@ class BackwardsCompatibleClientV2(object): self.client = Client(directory, key=key, net=net) else: self.client = ClientV2(directory, net=net) + self.orderr = None def __getattr__(self, name): if name in vars(self.client): @@ -705,6 +706,19 @@ class BackwardsCompatibleClientV2(object): regr = regr.update(terms_of_service_agreed=True) return self.client.new_account(regr) + def request_authorizations(self, csr_pem): + if self.acme_version == 1: + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) + authorizations = [] + for domain in dnsNames: + authorizations.append(self.client.request_domain_challenges(domain)) + return authorizations + else: + self.orderr = self.client.new_order(csr_pem) + return self.orderr.authorizations + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 5f520cbcb..4f88199e3 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,10 +48,10 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def get_authorizations(self, domains, best_effort=False): + def get_authorizations(self, csr_pem, best_effort=False): """Retrieve all authorizations for challenges. - :param list domains: Domains for authorization + :param list csr_pem: CSR containing domains for authorization :param bool best_effort: Whether or not all authorizations are required (this is useful in renewal) @@ -62,8 +62,10 @@ class AuthHandler(object): authorizations """ - for domain in domains: - self.authzr[domain] = self.acme.request_domain_challenges(domain) + authzrs = self.acme.request_authorizations(csr_pem) + for authzr in authzrs: + self.authzr[authzr.body.identifier.value] = authzr + domains = self.authzr.keys() self._choose_challenges(domains) config = zope.component.getUtility(interfaces.IConfig) diff --git a/certbot/client.py b/certbot/client.py index 67ee8f7fa..8e3ec6c62 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,13 +235,9 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, domains, csr, authzr=None): + def obtain_certificate_from_csr(self, csr, authzr=None): """Obtain certificate. - Internal function with precondition that `domains` are - consistent with identifiers present in the `csr`. - - :param list domains: Domain names. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. @@ -261,10 +257,10 @@ class Client(object): if self.account.regr is None: raise errors.Error("Please register with the ACME server first.") - logger.debug("CSR: %s, domains: %s", csr, domains) + logger.debug("CSR: %s", csr) if authzr is None: - authzr = self.auth_handler.get_authorizations(domains) + authzr = self.auth_handler.get_authorizations(csr) certr = self.acme.request_issuance( jose.ComparableX509( @@ -307,13 +303,6 @@ class Client(object): :rtype: tuple """ - authzr = self.auth_handler.get_authorizations( - domains, - self.config.allow_subset_of_names) - - auth_domains = set(a.body.identifier.value for a in authzr) - domains = [d for d in domains if d in auth_domains] - # Create CSR from names if self.config.dry_run: key = util.Key(file=None, @@ -326,8 +315,12 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) + authzr = self.auth_handler.get_authorizations( + csr, + self.config.allow_subset_of_names) + certr, chain = self.obtain_certificate_from_csr( - domains, csr, authzr=authzr) + csr, authzr=authzr) return certr, chain, key, csr diff --git a/certbot/main.py b/certbot/main.py index ff3758985..d01f68920 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1064,7 +1064,7 @@ def _csr_get_and_save_cert(config, le_client): """ csr, _ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr) + certr, chain = le_client.obtain_certificate_from_csr(csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) From eaf739184cf517d5a3f5103caa072bb0cd39b4e9 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:29:42 -0800 Subject: [PATCH 03/56] pass pem to auth_handler --- certbot/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 8e3ec6c62..d7d2acb14 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -260,7 +260,7 @@ class Client(object): logger.debug("CSR: %s", csr) if authzr is None: - authzr = self.auth_handler.get_authorizations(csr) + authzr = self.auth_handler.get_authorizations(csr.data) certr = self.acme.request_issuance( jose.ComparableX509( @@ -316,7 +316,7 @@ class Client(object): csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) authzr = self.auth_handler.get_authorizations( - csr, + csr.data, self.config.allow_subset_of_names) certr, chain = self.obtain_certificate_from_csr( From ea2022588b4d95f1d14849b918b2cd3788cc6084 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:32:49 -0800 Subject: [PATCH 04/56] add docstring --- acme/acme/client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/acme/acme/client.py b/acme/acme/client.py index 6b4d65233..e7cf016bb 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -707,6 +707,16 @@ class BackwardsCompatibleClientV2(object): return self.client.new_account(regr) def request_authorizations(self, csr_pem): + """Request authorizations for the domains in csr_pem. + + Calls request_domain_challenges for each domain for V1, and + calls new_order and saves the result for V2. + + :param str csr_pem: A CSR in PEM format. + + :returns: List of Authorization Resources. + :rtype: list of `.AuthorizationResource` + """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) # pylint: disable=protected-access From 20d0b91c710bd8110aa5ee23082f45583e7789a4 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 17:35:10 -0800 Subject: [PATCH 05/56] switch interface to new_order and remove best_effort flag --- acme/acme/client.py | 18 ++++++++---------- certbot/auth_handler.py | 27 ++++++++++----------------- certbot/client.py | 15 ++++----------- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index e7cf016bb..1838fab42 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -671,7 +671,6 @@ class BackwardsCompatibleClientV2(object): self.client = Client(directory, key=key, net=net) else: self.client = ClientV2(directory, net=net) - self.orderr = None def __getattr__(self, name): if name in vars(self.client): @@ -706,16 +705,16 @@ class BackwardsCompatibleClientV2(object): regr = regr.update(terms_of_service_agreed=True) return self.client.new_account(regr) - def request_authorizations(self, csr_pem): - """Request authorizations for the domains in csr_pem. + def new_order(self, csr_pem): + """Request a new Order object from the server. - Calls request_domain_challenges for each domain for V1, and - calls new_order and saves the result for V2. + If using ACMEv1, returns a dummy OrderResource with only + the authorizations field filled in. :param str csr_pem: A CSR in PEM format. - :returns: List of Authorization Resources. - :rtype: list of `.AuthorizationResource` + :returns: The newly created order. + :rtype: OrderResource """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) @@ -724,10 +723,9 @@ class BackwardsCompatibleClientV2(object): authorizations = [] for domain in dnsNames: authorizations.append(self.client.request_domain_challenges(domain)) - return authorizations + return messages.OrderResource(authorizations=authorizations) else: - self.orderr = self.client.new_order(csr_pem) - return self.orderr.authorizations + return self.client.new_order(csr_pem) def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 4f88199e3..825513329 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,12 +48,11 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def get_authorizations(self, csr_pem, best_effort=False): + def handle_authorizations(self, orderr): """Retrieve all authorizations for challenges. - :param list csr_pem: CSR containing domains for authorization - :param bool best_effort: Whether or not all authorizations are - required (this is useful in renewal) + :param acme.messages.OrderResource orderr: must have + authorizations filled in :returns: List of authorization resources :rtype: list @@ -62,7 +61,7 @@ class AuthHandler(object): authorizations """ - authzrs = self.acme.request_authorizations(csr_pem) + authzrs = orderr.authorizations for authzr in authzrs: self.authzr[authzr.body.identifier.value] = authzr domains = self.authzr.keys() @@ -80,7 +79,7 @@ class AuthHandler(object): 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp, best_effort) + self._respond(resp) # Just make sure all decisions are complete. self.verify_authzr_complete() @@ -124,7 +123,7 @@ class AuthHandler(object): return resp - def _respond(self, resp, best_effort): + def _respond(self, resp): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -137,7 +136,7 @@ class AuthHandler(object): # Check for updated status... try: - self._poll_challenges(chall_update, best_effort) + self._poll_challenges(chall_update) finally: # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) @@ -169,7 +168,7 @@ class AuthHandler(object): return active_achalls def _poll_challenges( - self, chall_update, best_effort, min_sleep=3, max_rounds=15): + self, chall_update, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" dom_to_check = set(chall_update.keys()) comp_domains = set() @@ -190,14 +189,8 @@ class AuthHandler(object): chall_update[domain].remove(achall) # We failed some challenges... damage control else: - if best_effort: - comp_domains.add(domain) - logger.warning( - "Challenge failed for domain %s", - domain) - else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) + all_failed_achalls.update( + updated for _, updated in failed_achalls) if all_failed_achalls: _report_failed_challs(all_failed_achalls) diff --git a/certbot/client.py b/certbot/client.py index d7d2acb14..61e9db635 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,14 +235,12 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr, authzr=None): + def obtain_certificate_from_csr(self, csr): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. - :param list authzr: List of - :class:`acme.messages.AuthorizationResource` :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -259,8 +257,8 @@ class Client(object): logger.debug("CSR: %s", csr) - if authzr is None: - authzr = self.auth_handler.get_authorizations(csr.data) + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr) certr = self.acme.request_issuance( jose.ComparableX509( @@ -315,12 +313,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - authzr = self.auth_handler.get_authorizations( - csr.data, - self.config.allow_subset_of_names) - - certr, chain = self.obtain_certificate_from_csr( - csr, authzr=authzr) + certr, chain = self.obtain_certificate_from_csr(csr) return certr, chain, key, csr From 68e24a8ea7eeb405592e40ed340baff6f3f3821b Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 17:59:51 -0800 Subject: [PATCH 06/56] start test updates --- certbot/tests/auth_handler_test.py | 54 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 32c4c0d3b..d424e59ca 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -57,8 +57,8 @@ class ChallengeFactoryTest(unittest.TestCase): errors.Error, self.handler._challenge_factory, "failure.com", [0]) -class GetAuthorizationsTest(unittest.TestCase): - """get_authorizations test. +class HandleAuthorizationsTest(unittest.TestCase): + """handle_authorizations test. This tests everything except for all functions under _poll_challenges. @@ -92,12 +92,11 @@ class GetAuthorizationsTest(unittest.TestCase): @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_name1_tls_sni_01_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) - mock_poll.side_effect = self._validate_all - authzr = self.handler.get_authorizations(["0"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) @@ -115,14 +114,13 @@ class GetAuthorizationsTest(unittest.TestCase): @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES, combos=False) - mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = self.handler.get_authorizations(["0"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -146,7 +144,11 @@ class GetAuthorizationsTest(unittest.TestCase): mock_poll.side_effect = self._validate_all - authzr = self.handler.get_authorizations(["0", "1", "2"]) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -169,31 +171,33 @@ class GetAuthorizationsTest(unittest.TestCase): def test_debug_challenges(self, mock_poll): zope.component.provideUtility( mock.Mock(debug_challenges=True), interfaces.IConfig) - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) mock_poll.side_effect = self._validate_all - self.handler.get_authorizations(["0"]) + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(self.mock_display.notification.call_count, 1) def test_perform_failure(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + self.mock_auth.perform.side_effect = errors.AuthorizationError self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) def test_no_domains(self): - self.assertRaises(errors.AuthorizationError, self.handler.get_authorizations, []) + mock_order = mock.MagicMock(authorizations=[]) + self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_preferred_challenge_choice(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) @@ -201,20 +205,20 @@ class GetAuthorizationsTest(unittest.TestCase): self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - self.handler.get_authorizations(["0"]) + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") def test_preferred_challenges_not_supported(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) self.handler.pref_challs.append(challenges.HTTP01.typ) self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1, unused_2): + def _validate_all(self, unused_1): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] self.handler.authzr[dom] = acme_util.gen_authzr( From 9c84fe1144cc5ccec5fc9cfcdf7c5adf8cfdf15c Mon Sep 17 00:00:00 2001 From: Matt Christian Date: Sun, 18 Feb 2018 15:45:22 -0600 Subject: [PATCH 07/56] Add override class for ID="ol" AKA Oracle Linux Server, a clone of CentOS/RHEL. --- certbot-apache/certbot_apache/entrypoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-apache/certbot_apache/entrypoint.py b/certbot-apache/certbot_apache/entrypoint.py index 4267398d5..6f1443507 100644 --- a/certbot-apache/certbot_apache/entrypoint.py +++ b/certbot-apache/certbot_apache/entrypoint.py @@ -17,6 +17,7 @@ OVERRIDE_CLASSES = { "centos": override_centos.CentOSConfigurator, "centos linux": override_centos.CentOSConfigurator, "fedora": override_centos.CentOSConfigurator, + "ol": override_centos.CentOSConfigurator, "red hat enterprise linux server": override_centos.CentOSConfigurator, "rhel": override_centos.CentOSConfigurator, "amazon": override_centos.CentOSConfigurator, From d6b4e2001b404b1f9bd4c3929e888726d583ee57 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 13:19:04 -0800 Subject: [PATCH 08/56] put back in best_effort code, with a todo for actually supporting it in ACMEv2 --- certbot/auth_handler.py | 22 +++++++++++++++------- certbot/client.py | 11 ++++++++--- certbot/tests/auth_handler_test.py | 2 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 825513329..662cadc65 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,11 +48,13 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def handle_authorizations(self, orderr): + def handle_authorizations(self, orderr, best_effort=False): """Retrieve all authorizations for challenges. :param acme.messages.OrderResource orderr: must have authorizations filled in + :param bool best_effort: Whether or not all authorizations are + required (this is useful in renewal) :returns: List of authorization resources :rtype: list @@ -79,7 +81,7 @@ class AuthHandler(object): 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp) + self._respond(resp, best_effort) # Just make sure all decisions are complete. self.verify_authzr_complete() @@ -123,7 +125,7 @@ class AuthHandler(object): return resp - def _respond(self, resp): + def _respond(self, resp, best_effort): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -136,7 +138,7 @@ class AuthHandler(object): # Check for updated status... try: - self._poll_challenges(chall_update) + self._poll_challenges(chall_update, best_effort) finally: # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) @@ -168,7 +170,7 @@ class AuthHandler(object): return active_achalls def _poll_challenges( - self, chall_update, min_sleep=3, max_rounds=15): + self, chall_update, best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" dom_to_check = set(chall_update.keys()) comp_domains = set() @@ -189,8 +191,14 @@ class AuthHandler(object): chall_update[domain].remove(achall) # We failed some challenges... damage control else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) + if best_effort: + comp_domains.add(domain) + logger.warning( + "Challenge failed for domain %s", + domain) + else: + all_failed_achalls.update( + updated for _, updated in failed_achalls) if all_failed_achalls: _report_failed_challs(all_failed_achalls) diff --git a/certbot/client.py b/certbot/client.py index 61e9db635..5feea662d 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,12 +235,13 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr): + def obtain_certificate_from_csr(self, csr, best_effort=False): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. + :param bool best_effort: Whether or not all authorizations are required :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -258,7 +259,11 @@ class Client(object): logger.debug("CSR: %s", csr) orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr) + authzr = self.auth_handler.handle_authorizations(orderr, best_effort) + if best_effort: + # TODO: check if we passed all authorizations, and if not, + # create a new order and try again, possibly in a loop + pass certr = self.acme.request_issuance( jose.ComparableX509( @@ -313,7 +318,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - certr, chain = self.obtain_certificate_from_csr(csr) + certr, chain = self.obtain_certificate_from_csr(csr, self.config.allow_subset_of_names) return certr, chain, key, csr diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index d424e59ca..ea8b006c4 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -218,7 +218,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1): + def _validate_all(self, unused_1, unused_2): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] self.handler.authzr[dom] = acme_util.gen_authzr( From 11f2f1e576243a255afbcd166b042cab2e2f1c4b Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 13:20:41 -0800 Subject: [PATCH 09/56] remove extra spaces --- certbot/auth_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 662cadc65..47d806b94 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -194,8 +194,8 @@ class AuthHandler(object): if best_effort: comp_domains.add(domain) logger.warning( - "Challenge failed for domain %s", - domain) + "Challenge failed for domain %s", + domain) else: all_failed_achalls.update( updated for _, updated in failed_achalls) From a0e84e65ce9cfc041f341a6bedf36525e49fa1b2 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:29:04 -0800 Subject: [PATCH 10/56] auth_handler tests are happy --- certbot/tests/auth_handler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index ea8b006c4..3633b673d 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -118,7 +118,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) mock_order = mock.MagicMock(authorizations=[authzr]) authzr = self.handler.handle_authorizations(mock_order) From 76a0cbf9c23b9827c4bf94e9a6521a4bf049a466 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:43:12 -0800 Subject: [PATCH 11/56] client tests passing --- certbot/tests/client_test.py | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a9a87b80b..b6cbca367 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -141,16 +141,17 @@ class ClientTest(ClientTestCommon): def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() - self.client.auth_handler.get_authorizations.return_value = [None] + self.client.auth_handler.handle_authorizations.return_value = [None] self.acme.request_issuance.return_value = mock.sentinel.certr self.acme.fetch_chain.return_value = mock.sentinel.chain + self.acme.new_order.return_value = mock.sentinel.orderr def _check_obtain_certificate(self): - self.client.auth_handler.get_authorizations.assert_called_once_with( - self.eg_domains, + self.client.auth_handler.handle_authorizations.assert_called_once_with( + mock.sentinel.orderr, self.config.allow_subset_of_names) - authzr = self.client.auth_handler.get_authorizations() + authzr = self.client.auth_handler.handle_authorizations() self.acme.request_issuance.assert_called_once_with( jose.ComparableX509(OpenSSL.crypto.load_certificate_request( @@ -167,31 +168,19 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, test_csr, - authzr=authzr)) + best_effort=False)) # and that the cert was obtained correctly self._check_obtain_certificate() - # Test for authzr=None - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=None)) - auth_handler.get_authorizations.assert_called_with(self.eg_domains) - # Test for no auth_handler self.client.auth_handler = None self.assertRaises( errors.Error, self.client.obtain_certificate_from_csr, - self.eg_domains, test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) @@ -204,13 +193,10 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="der", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=authzr)) + test_csr)) self.assertEqual(1, mock_get_utility().notification.call_count) @test_util.patch_get_utility() @@ -220,13 +206,10 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="der", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, - self.eg_domains, - test_csr, - authzr=authzr) + test_csr) self.assertEqual(1, mock_get_utility().notification.call_count) @mock.patch("certbot.client.crypto_util") @@ -276,7 +259,7 @@ class ClientTest(ClientTestCommon): identifier=mock.MagicMock( value=domain)))) - self.client.auth_handler.get_authorizations.return_value = authzr + self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): result = self.client.obtain_certificate(self.eg_domains) From 3dfeb483ee853333b11e829ef7d985ebf5ad7269 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:49:23 -0800 Subject: [PATCH 12/56] lint --- certbot/tests/client_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index b6cbca367..570080e3b 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -166,7 +166,6 @@ class ClientTest(ClientTestCommon): mock_logger): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), @@ -191,7 +190,6 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), @@ -204,7 +202,6 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertRaises( acme_errors.Error, From d6af978472d7519b615e8894903383610ab41269 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:52:11 -0800 Subject: [PATCH 13/56] remove if/pass --- certbot/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 5feea662d..65e85a159 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -260,10 +260,8 @@ class Client(object): orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr, best_effort) - if best_effort: - # TODO: check if we passed all authorizations, and if not, - # create a new order and try again, possibly in a loop - pass + # TODO: check if we passed all authorizations, and if not, + # create a new order and try again, possibly in a loop certr = self.acme.request_issuance( jose.ComparableX509( From d29c637bf94cf083ef3dc1648bed3697229b8025 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:36:35 -0800 Subject: [PATCH 14/56] support best_effort --- certbot/client.py | 29 +++++++++++++++++++++-------- certbot/tests/client_test.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 65e85a159..d00055eae 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,13 +235,13 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr, best_effort=False): + def obtain_certificate_from_csr(self, csr, orderr=None): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. - :param bool best_effort: Whether or not all authorizations are required + :param acme.messages.OrderResource orderr: contains authzrs :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -258,10 +258,12 @@ class Client(object): logger.debug("CSR: %s", csr) - orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr, best_effort) - # TODO: check if we passed all authorizations, and if not, - # create a new order and try again, possibly in a loop + if orderr is None: + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr) + else: + authzr = orderr.authorizations + certr = self.acme.request_issuance( jose.ComparableX509( @@ -316,9 +318,20 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - certr, chain = self.obtain_certificate_from_csr(csr, self.config.allow_subset_of_names) + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) + auth_domains = set(a.body.identifier.value for a in authzr) + successful_domains = [d for d in domains if d in auth_domains] - return certr, chain, key, csr + if successful_domains != domains: + if not self.config.dry_run: + # TODO: delete keys + pass + return self.obtain_certificate(successful_domains) + else: + certr, chain = self.obtain_certificate_from_csr(csr, orderr) + + return certr, chain, key, csr # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 570080e3b..376bf5a90 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -134,6 +134,7 @@ class ClientTest(ClientTestCommon): self.config.allow_subset_of_names = False self.config.dry_run = False self.eg_domains = ["example.com", "www.example.com"] + self.eg_order = mock.MagicMock(authorizations=[None]) def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] @@ -144,11 +145,11 @@ class ClientTest(ClientTestCommon): self.client.auth_handler.handle_authorizations.return_value = [None] self.acme.request_issuance.return_value = mock.sentinel.certr self.acme.fetch_chain.return_value = mock.sentinel.chain - self.acme.new_order.return_value = mock.sentinel.orderr + self.acme.new_order.return_value = self.eg_order def _check_obtain_certificate(self): self.client.auth_handler.handle_authorizations.assert_called_once_with( - mock.sentinel.orderr, + self.eg_order, self.config.allow_subset_of_names) authzr = self.client.auth_handler.handle_authorizations() @@ -166,15 +167,26 @@ class ClientTest(ClientTestCommon): mock_logger): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, - best_effort=False)) + orderr=orderr)) # and that the cert was obtained correctly self._check_obtain_certificate() + # Test for orderr=None + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr( + test_csr, + orderr=None)) + auth_handler.handle_authorizations.assert_called_with(self.eg_order) + # Test for no auth_handler self.client.auth_handler = None self.assertRaises( @@ -190,11 +202,15 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - test_csr)) + test_csr, + orderr=orderr)) self.assertEqual(1, mock_get_utility().notification.call_count) @test_util.patch_get_utility() @@ -202,11 +218,15 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, - test_csr) + test_csr, + orderr=orderr) self.assertEqual(1, mock_get_utility().notification.call_count) @mock.patch("certbot.client.crypto_util") @@ -256,6 +276,7 @@ class ClientTest(ClientTestCommon): identifier=mock.MagicMock( value=domain)))) + self.eg_order.authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): From 7c073dbcaf6d6789da01734f0a32951b18b3e4d6 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:38:18 -0800 Subject: [PATCH 15/56] lint --- certbot/tests/client_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 376bf5a90..f0ef077b2 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,7 +170,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( @@ -205,7 +204,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( @@ -221,7 +219,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, From 051664a142a8c77121c79cb07307379a63d76874 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:39:30 -0800 Subject: [PATCH 16/56] lint --- certbot/tests/client_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f0ef077b2..f10053616 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -201,7 +201,6 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) self.assertEqual( @@ -216,7 +215,6 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) self.assertRaises( From d5a90c5a6e58a068c0fbd8bf800f9953425a9693 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:43:27 -0800 Subject: [PATCH 17/56] delete key and csr before trying again --- certbot/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index d00055eae..404e1e0d9 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -325,8 +325,8 @@ class Client(object): if successful_domains != domains: if not self.config.dry_run: - # TODO: delete keys - pass + os.remove(key.file) + os.remove(csr.file) return self.obtain_certificate(successful_domains) else: certr, chain = self.obtain_certificate_from_csr(csr, orderr) From 26bcaff85cae0b38280f576ac732bd5a4744d2f8 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:59:58 -0800 Subject: [PATCH 18/56] add test for new_order for v2 --- acme/acme/client_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 11516c02f..1ba41cd7d 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -161,6 +161,21 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().register.assert_called_once_with(self.new_reg) mock_client().agree_to_tos.assert_not_called() + def test_new_order_v1(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + + def test_new_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_csr_pem = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_order(mock_csr_pem) + mock_client().new_order.assert_called_once_with(mock_csr_pem) + + + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" From 65d0b9674cb4008e624c1a58cc7c8e4ee91fcdb6 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:01:35 -0800 Subject: [PATCH 19/56] Fix client test --- certbot/tests/client_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f10053616..6b90eab83 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,6 +170,7 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) + auth_handler.handle_authorizations(orderr) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( From a7eadf88629d9cbba611731b9fe555bc0ba5cb84 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:08:46 -0800 Subject: [PATCH 20/56] add new order test for v1 --- acme/acme/client_test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1ba41cd7d..773d59aa2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -161,10 +161,18 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().register.assert_called_once_with(self.new_reg) mock_client().agree_to_tos.assert_not_called() - def test_new_order_v1(self): + @mock.patch('OpenSSL.crypto.load_certificate_request') + @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names') + def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names, + mock_load_certificate_request): self.response.json.return_value = DIRECTORY_V1.to_json() + mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com'] + mock_csr_pem = mock.MagicMock() with mock.patch('acme.client.Client') as mock_client: + mock_client().request_domain_challenges.return_value = mock.sentinel.auth client = self._init() + orderr = client.new_order(mock_csr_pem) + self.assertEqual(orderr.authorizations, [mock.sentinel.auth, mock.sentinel.auth]) def test_new_order_v2(self): self.response.json.return_value = DIRECTORY_V2.to_json() From dea43e90b629f629a19e6f865fbac2930421a733 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:11:36 -0800 Subject: [PATCH 21/56] lint --- acme/acme/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 773d59aa2..1b33ca5d7 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -164,7 +164,7 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): @mock.patch('OpenSSL.crypto.load_certificate_request') @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names') def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names, - mock_load_certificate_request): + unused_mock_load_certificate_request): self.response.json.return_value = DIRECTORY_V1.to_json() mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com'] mock_csr_pem = mock.MagicMock() From df50f2d5fa3ddd70cfcd01cf63c305431c577d46 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:12:15 -0800 Subject: [PATCH 22/56] client test --- certbot/tests/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 6b90eab83..ecd77bdeb 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,7 +170,7 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - auth_handler.handle_authorizations(orderr) + auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( From d13a4ed18da3f2a0b8f88076bf15f778337259ea Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:50:18 -0800 Subject: [PATCH 23/56] add tests for if partial auth success --- certbot/tests/client_test.py | 47 ++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index ecd77bdeb..f4a8a5c8a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -147,10 +147,13 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.return_value = mock.sentinel.chain self.acme.new_order.return_value = self.eg_order - def _check_obtain_certificate(self): - self.client.auth_handler.handle_authorizations.assert_called_once_with( - self.eg_order, - self.config.allow_subset_of_names) + def _check_obtain_certificate(self, auth_count=1): + if auth_count == 1: + self.client.auth_handler.handle_authorizations.assert_called_once_with( + self.eg_order, + self.config.allow_subset_of_names) + else: + self.assertEqual(self.client.auth_handler.handle_authorizations.call_count, auth_count) authzr = self.client.auth_handler.handle_authorizations() @@ -238,6 +241,21 @@ class ClientTest(ClientTestCommon): mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) + @mock.patch("certbot.client.crypto_util") + @mock.patch("os.remove") + def test_obtain_certificate_partial_success(self, mock_remove, mock_crypto_util): + csr = util.CSR(form="pem", file=mock.sentinel.csr_file, data=CSR_SAN) + key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) + mock_crypto_util.init_save_csr.return_value = csr + mock_crypto_util.init_save_key.return_value = key + + authzr = self._authzr_from_domains(["example.com"]) + self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) + + self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) + self.assertEqual(mock_crypto_util.init_save_csr.call_count, 2) + self.assertEqual(mock_remove.call_count, 2) + @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.acme_crypto_util") def test_obtain_certificate_dry_run(self, mock_acme_crypto, mock_crypto): @@ -255,22 +273,25 @@ class ClientTest(ClientTestCommon): mock_crypto.init_save_key.assert_not_called() mock_crypto.init_save_csr.assert_not_called() - def _test_obtain_certificate_common(self, key, csr): - self._mock_obtain_certificate() - - # return_value is essentially set to (None, None) in - # _mock_obtain_certificate(), which breaks this test. - # Thus fixed by the next line. - + def _authzr_from_domains(self, domains): authzr = [] # domain ordering should not be affected by authorization order - for domain in reversed(self.eg_domains): + for domain in reversed(domains): authzr.append( mock.MagicMock( body=mock.MagicMock( identifier=mock.MagicMock( value=domain)))) + return authzr + + def _test_obtain_certificate_common(self, key, csr, authzr_ret=None, auth_count=1): + self._mock_obtain_certificate() + + # return_value is essentially set to (None, None) in + # _mock_obtain_certificate(), which breaks this test. + # Thus fixed by the next line. + authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr @@ -281,7 +302,7 @@ class ClientTest(ClientTestCommon): self.assertEqual( result, (mock.sentinel.certr, mock.sentinel.chain, key, csr)) - self._check_obtain_certificate() + self._check_obtain_certificate(auth_count) @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') From ea3b78e3c9ff11da1d6919f2c5c36d53623512d3 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 20 Feb 2018 18:53:48 -0800 Subject: [PATCH 24/56] update order object with returned authorizations (#5598) --- certbot/client.py | 6 +++--- certbot/tests/client_test.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 404e1e0d9..dd11f2204 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -261,9 +261,8 @@ class Client(object): if orderr is None: orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr) - else: - authzr = orderr.authorizations - + orderr = orderr.update(authorizations=authzr) + authzr = orderr.authorizations certr = self.acme.request_issuance( jose.ComparableX509( @@ -320,6 +319,7 @@ class Client(object): orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) + orderr = orderr.update(authorizations=authzr) auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f4a8a5c8a..a65341692 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -294,6 +294,7 @@ class ClientTest(ClientTestCommon): authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr + self.eg_order.update().authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): From f1b7017c0c632054c61a231f8cd83db092d063fa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Feb 2018 16:20:45 -0800 Subject: [PATCH 25/56] Finish dropping Python 2.6 and 3.3 support * Undo letsencrypt-auto changes * Remove ordereddict import * Add Python 3.4 tests to replace 3.3 * Add python_requires * update pipstrap --- .travis.yml | 4 + acme/setup.py | 1 + certbot-apache/setup.py | 1 + certbot-compatibility-test/setup.py | 1 + certbot-dns-cloudflare/setup.py | 1 + certbot-dns-cloudxns/setup.py | 1 + certbot-dns-digitalocean/setup.py | 1 + certbot-dns-dnsimple/setup.py | 1 + certbot-dns-dnsmadeeasy/setup.py | 1 + certbot-dns-google/setup.py | 1 + certbot-dns-luadns/setup.py | 1 + certbot-dns-nsone/setup.py | 1 + certbot-dns-rfc2136/setup.py | 1 + certbot-dns-route53/setup.py | 1 + certbot-nginx/setup.py | 1 + certbot/plugins/disco.py | 8 +- certbot/util.py | 8 +- letsencrypt-auto-source/letsencrypt-auto | 124 ++++++++++++------ .../letsencrypt-auto.template | 33 +++-- letsencrypt-auto-source/pieces/pipstrap.py | 91 ++++++++----- letshelp-certbot/setup.py | 1 + setup.py | 1 + 22 files changed, 192 insertions(+), 92 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1077d99d9..42b8d679d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,10 @@ matrix: env: TOXENV=py27-oldest sudo: required services: docker + - python: "3.4" + env: TOXENV=py34 + sudo: required + services: docker - python: "3.6" env: TOXENV=py36 sudo: required diff --git a/acme/setup.py b/acme/setup.py index ba5c8e6fb..1f16f3b99 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -45,6 +45,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index d7c223a0a..86b0c646e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 7e1b059e2..861921ef7 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -34,6 +34,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d619f1872..6db6cc48f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 5d14f3e29..bf337c3d0 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index ce8fedd46..12d55f660 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 06af16759..79c93c942 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 7c0f3ed86..5d0970af1 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index de881ad84..cdfa205aa 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -36,6 +36,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 0d580b7ee..6c0dfb68f 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index c0ba11470..09a4e2cf7 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 5161e7a94..06efc373d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 09f8a7d52..8bd157166 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 37c477ef6..58f687aea 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 37baf98f7..5a7e07ec0 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -5,6 +5,8 @@ import logging import pkg_resources import six +from collections import OrderedDict + import zope.interface import zope.interface.verify @@ -12,12 +14,6 @@ from certbot import constants from certbot import errors from certbot import interfaces -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) diff --git a/certbot/util.py b/certbot/util.py index b7e60a225..b81799373 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -16,18 +16,14 @@ import stat import subprocess import sys +from collections import OrderedDict + import configargparse from certbot import constants from certbot import errors from certbot import lock -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index aed15a8ef..85659cfad 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -781,11 +781,20 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -956,10 +965,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -1220,6 +1237,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -1253,33 +1271,32 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 0 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), +# Pip has no dependencies, as it vendors everything: +PIP_PACKAGE = [ + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] + + +OTHER_PACKAGES = maybe_argparse + [ # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -1300,12 +1317,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -1315,8 +1333,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -1329,6 +1348,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -1336,17 +1373,24 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # We download and install pip first, then the rest, to avoid the bug + # https://github.com/certbot/certbot/issues/4938. + pip_downloads, other_downloads = [ + [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in packages] + for packages in (PIP_PACKAGE, OTHER_PACKAGES)] + for downloads in (pip_downloads, other_downloads): + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it + # otherwise sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index b3d6ab740..618e8f6bd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -320,11 +320,20 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -495,10 +504,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 78491b5e3..ed55b37e9 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -23,6 +23,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -56,33 +57,32 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 0 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), +# Pip has no dependencies, as it vendors everything: +PIP_PACKAGE = [ + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] + + +OTHER_PACKAGES = maybe_argparse + [ # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -103,12 +103,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -118,8 +119,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -132,6 +134,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -139,17 +159,24 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # We download and install pip first, then the rest, to avoid the bug + # https://github.com/certbot/certbot/issues/4938. + pip_downloads, other_downloads = [ + [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in packages] + for packages in (PIP_PACKAGE, OTHER_PACKAGES)] + for downloads in (pip_downloads, other_downloads): + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it + # otherwise sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 7c8c39068..b5be07a59 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: System Administrators', diff --git a/setup.py b/setup.py index e3824a7f7..f314449e6 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Console', From f3b23662f1fddedcea702c6ea5b54d5f97d1b0a8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 21 Feb 2018 20:52:04 -0800 Subject: [PATCH 26/56] Don't error immediately on wildcards. (#5600) --- certbot/tests/display/ops_test.py | 4 ++-- certbot/tests/main_test.py | 7 ++----- certbot/util.py | 10 ---------- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 57d82f839..c4f58ba7c 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -300,8 +300,8 @@ class ChooseNamesTest(unittest.TestCase): from certbot.display.ops import get_valid_domains all_valid = ["example.com", "second.example.com", "also.example.com", "under_score.example.com", - "justtld"] - all_invalid = ["öóòps.net", "*.wildcard.com", "uniçodé.com"] + "justtld", "*.wildcard.com"] + all_invalid = ["öóòps.net", "uniçodé.com"] two_valid = ["example.com", "úniçøde.com", "also.example.com"] self.assertEqual(get_valid_domains(all_valid), all_valid) self.assertEqual(get_valid_domains(all_invalid), []) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 518653a53..c31a3fb33 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,3 +1,4 @@ +# coding=utf-8 """Tests for certbot.main.""" # pylint: disable=too-many-lines from __future__ import print_function @@ -939,10 +940,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertRaises(errors.ConfigurationError, self._call, ['-d', (('a' * 50) + '.') * 10]) - # Wildcard - self.assertRaises(errors.ConfigurationError, - self._call, - ['-d', '*.wildcard.tld']) # Bare IP address (this is actually a different error message now) self.assertRaises(errors.ConfigurationError, @@ -1232,7 +1229,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def test_renew_with_bad_domain(self): renewalparams = {'authenticator': 'webroot'} - names = ['*.example.com'] + names = ['uniçodé.com'] self._test_renew_common(renewalparams=renewalparams, error_expected=True, names=names, assert_oc_called=False) diff --git a/certbot/util.py b/certbot/util.py index b7e60a225..70f402a72 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -552,16 +552,6 @@ def enforce_domain_sanity(domain): :returns: The domain cast to `str`, with ASCII-only contents :rtype: str """ - if isinstance(domain, six.text_type): - wildcard_marker = u"*." - else: - wildcard_marker = b"*." - - # Check if there's a wildcard domain - if domain.startswith(wildcard_marker): - raise errors.ConfigurationError( - "Wildcard domains are not supported: {0}".format(domain)) - # Unicode try: if isinstance(domain, six.binary_type): From c3659c300b0720d29939331c1c7c86586fd629ae Mon Sep 17 00:00:00 2001 From: Marcus LaFerrera Date: Thu, 22 Feb 2018 13:09:06 -0500 Subject: [PATCH 27/56] Return str rather than bytes (#5585) * Return str rather than bytes Project id is returned as bytes, which causes issues when constructing the google cloud API url, converting `b'PROJECT_ID'` to `b%27PROJECT_ID%27` causing the request to fail. * Ensure we handle both bytes and str types * project_id should be a str or bytes, not int --- certbot-dns-google/certbot_dns_google/dns_google.py | 5 ++++- certbot-dns-google/certbot_dns_google/dns_google_test.py | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index 37fd6b0de..cea754c06 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -224,4 +224,7 @@ class _GoogleClient(object): if r.status != 200: raise ValueError("Invalid status code: {0}".format(r)) - return content + if isinstance(content, bytes): + return content.decode() + else: + return content diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 85649fc7f..53f84dd6e 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -223,9 +223,13 @@ class GoogleClientTest(unittest.TestCase): response = DummyResponse() response.status = 200 - with mock.patch('httplib2.Http.request', return_value=(response, 1234)): + with mock.patch('httplib2.Http.request', return_value=(response, 'test-test-1')): project_id = _GoogleClient.get_project_id() - self.assertEqual(project_id, 1234) + self.assertEqual(project_id, 'test-test-1') + + with mock.patch('httplib2.Http.request', return_value=(response, b'test-test-1')): + project_id = _GoogleClient.get_project_id() + self.assertEqual(project_id, 'test-test-1') failed_response = DummyResponse() failed_response.status = 404 From 457269b0052f68b7e1b4a75414175a2e22f27ae6 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 22 Feb 2018 10:14:29 -0800 Subject: [PATCH 28/56] Add finalize_order to shim object, update Certbot to use it (#5601) * update order object with returned authorizations * major structure of finalize_order shim refactor * util methods and imports for finalize_order shim refactor * update certbot.tests.client_test.py * extraneous client_test imports * remove correct import * update renewal call * add test for acme.dump_pyopenssl_chain * Add test for certbot.crypto_util.cert_and_chain_from_fullchain * add tests for acme.client and change to fetch chain failure to TimeoutError * s/rytpe/rtype * remove ClientV1 passthrough * dump the wrapped cert * remove dead code * remove the correct dead code * support earlier mock --- acme/acme/client.py | 43 ++++++++++++++-- acme/acme/client_test.py | 71 ++++++++++++++++++++++++- acme/acme/crypto_util.py | 22 ++++++++ acme/acme/crypto_util_test.py | 28 ++++++++++ certbot/client.py | 81 +++++++++-------------------- certbot/crypto_util.py | 25 +++++---- certbot/main.py | 4 +- certbot/renewal.py | 5 +- certbot/tests/client_test.py | 86 +++++++++++-------------------- certbot/tests/crypto_util_test.py | 13 +++++ certbot/tests/util.py | 5 -- 11 files changed, 240 insertions(+), 143 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 1838fab42..97f529aae 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -677,9 +677,6 @@ class BackwardsCompatibleClientV2(object): return getattr(self.client, name) elif name in dir(ClientBase): return getattr(self.client, name) - # temporary, for breaking changes into smaller pieces - elif name in dir(Client): - return getattr(self.client, name) else: raise AttributeError() @@ -723,10 +720,48 @@ class BackwardsCompatibleClientV2(object): authorizations = [] for domain in dnsNames: authorizations.append(self.client.request_domain_challenges(domain)) - return messages.OrderResource(authorizations=authorizations) + return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem) else: return self.client.new_order(csr_pem) + def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ + if self.acme_version == 1: + csr_pem = orderr.csr_pem + certr = self.client.request_issuance( + jose.ComparableX509( + OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)), + orderr.authorizations) + + chain = None + while datetime.datetime.now() < deadline: + try: + chain = self.client.fetch_chain(certr) + break + except errors.Error: + time.sleep(1) + + if chain is None: + raise errors.TimeoutError( + 'Failed to fetch chain. You should not deploy the generated ' + 'certificate, please rerun the command for a new one.') + + cert = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) + chain = crypto_util.dump_pyopenssl_chain(chain) + + return orderr.update(fullchain_pem=(cert + chain)) + else: + return self.client.finalize_order(orderr, deadline) + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1b33ca5d7..acc5193ca 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -8,6 +8,7 @@ from six.moves import http_client # pylint: disable=import-error import josepy as jose import mock +import OpenSSL import requests from acme import challenges @@ -82,6 +83,29 @@ class ClientTestBase(unittest.TestCase): class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" + def setUp(self): + super(BackwardsCompatibleClientV2Test, self).setUp() + # contains a loaded cert + self.certr = messages.CertificateResource( + body=messages_test.CERT) + + loaded = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, CERT_SAN_PEM) + wrapped = jose.ComparableX509(loaded) + self.chain = [wrapped, wrapped] + + self.cert_pem = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped) + + single_chain = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, loaded) + self.chain_pem = single_chain + single_chain + + self.fullchain_pem = self.cert_pem + self.chain_pem + + self.orderr = messages.OrderResource( + csr_pem=CSR_SAN_PEM) + def _init(self): uri = 'http://www.letsencrypt-demo.org/directory' from acme.client import BackwardsCompatibleClientV2 @@ -109,8 +133,6 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client = self._init() self.assertEqual(client.directory, client.client.directory) self.assertEqual(client.key, KEY) - # delete this line once we finish migrating to new API: - self.assertEqual(client.register, client.client.register) self.assertEqual(client.update_registration, client.client.update_registration) self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') @@ -182,7 +204,52 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.new_order(mock_csr_pem) mock_client().new_order.assert_called_once_with(mock_csr_pem) + @mock.patch('acme.client.Client') + def test_finalize_order_v1_success(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + mock_client().fetch_chain.assert_called_once_with(self.certr) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_fetch_chain_error(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + mock_client().fetch_chain.side_effect = [errors.Error, self.chain] + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + self.assertEqual(mock_client().fetch_chain.call_count, 2) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_timeout(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + + deadline = deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + client = self._init() + self.assertRaises(errors.TimeoutError, client.finalize_order, + self.orderr, deadline) + + def test_finalize_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_orderr = mock.MagicMock() + mock_deadline = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.finalize_order(mock_orderr, mock_deadline) + mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline) class ClientTest(ClientTestBase): diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index a986721f0..f13c5109c 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -8,6 +8,8 @@ import socket import sys import OpenSSL +import josepy as jose + from acme import errors @@ -280,3 +282,23 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_pubkey(key) cert.sign(key, "sha256") return cert + +def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): + """Dump certificate chain into a bundle. + + :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in + :class:`josepy.util.ComparableX509`). + + """ + # XXX: returns empty string when no chain is available, which + # shuts up RenewableCert, but might not be the best solution... + + def _dump_cert(cert): + if isinstance(cert, jose.ComparableX509): + # pylint: disable=protected-access + cert = cert.wrapped + return OpenSSL.crypto.dump_certificate(filetype, cert) + + # assumes that OpenSSL.crypto.dump_certificate includes ending + # newline character + return b"".join(_dump_cert(cert) for cert in chain) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 14aaac8b5..e8dd3b20c 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -225,5 +225,33 @@ class MakeCSRTest(unittest.TestCase): self.assertEqual(len(must_staple_exts), 1, "Expected exactly one Must Staple extension") + +class DumpPyopensslChainTest(unittest.TestCase): + """Test for dump_pyopenssl_chain.""" + + @classmethod + def _call(cls, loaded): + # pylint: disable=protected-access + from acme.crypto_util import dump_pyopenssl_chain + return dump_pyopenssl_chain(loaded) + + def test_dump_pyopenssl_chain(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + length = sum( + len(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) + for cert in loaded) + self.assertEqual(len(self._call(loaded)), length) + + def test_dump_pyopenssl_chain_wrapped(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + wrap_func = jose.ComparableX509 + wrapped = [wrap_func(cert) for cert in loaded] + dump_func = OpenSSL.crypto.dump_certificate + length = sum(len(dump_func(OpenSSL.crypto.FILETYPE_PEM, cert)) for cert in loaded) + self.assertEqual(len(self._call(wrapped)), length) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/client.py b/certbot/client.py index dd11f2204..fc3848a5c 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -1,4 +1,5 @@ """Certbot client API.""" +import datetime import logging import os import platform @@ -11,7 +12,6 @@ import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util -from acme import errors as acme_errors from acme import messages import certbot @@ -243,8 +243,7 @@ class Client(object): than `authkey`. :param acme.messages.OrderResource orderr: contains authzrs - :returns: `.CertificateResource` and certificate chain (as - returned by `.fetch_chain`). + :returns: certificate and chain as PEM strings :rtype: tuple """ @@ -264,32 +263,9 @@ class Client(object): orderr = orderr.update(authorizations=authzr) authzr = orderr.authorizations - certr = self.acme.request_issuance( - jose.ComparableX509( - OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr.data)), - authzr) - - notify = zope.component.getUtility(interfaces.IDisplay).notification - retries = 0 - chain = None - - while retries <= 1: - if retries: - notify('Failed to fetch chain, please check your network ' - 'and continue', pause=True) - try: - chain = self.acme.fetch_chain(certr) - break - except acme_errors.Error: - logger.debug('Failed to fetch chain', exc_info=True) - retries += 1 - - if chain is None: - raise acme_errors.Error( - 'Failed to fetch chain. You should not deploy the generated ' - 'certificate, please rerun the command for a new one.') - - return certr, chain + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.acme.finalize_order(orderr, deadline) + return crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. @@ -298,10 +274,9 @@ class Client(object): :param list domains: domains to get a certificate - :returns: `.CertificateResource`, certificate chain (as - returned by `.fetch_chain`), and newly generated private key - (`.util.Key`) and DER-encoded Certificate Signing Request - (`.util.CSR`). + :returns: :returns: certificate as PEM string, chain as PEM string, + newly generated private key (`.util.Key`), and DER-encoded + Certificate Signing Request (`.util.CSR`). :rtype: tuple """ @@ -329,9 +304,9 @@ class Client(object): os.remove(csr.file) return self.obtain_certificate(successful_domains) else: - certr, chain = self.obtain_certificate_from_csr(csr, orderr) + cert, chain = self.obtain_certificate_from_csr(csr, orderr) - return certr, chain, key, csr + return cert, chain, key, csr # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): @@ -350,7 +325,7 @@ class Client(object): be obtained, or None if doing a successful dry run. """ - certr, chain, key, _ = self.obtain_certificate(domains) + cert, chain, key, _ = self.obtain_certificate(domains) if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): @@ -365,19 +340,16 @@ class Client(object): return None else: return storage.RenewableCert.new_lineage( - new_name, OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), - key.pem, crypto_util.dump_pyopenssl_chain(chain), + new_name, cert, + key.pem, chain, self.config) - def save_certificate(self, certr, chain_cert, + def save_certificate(self, cert_pem, chain_pem, cert_path, chain_path, fullchain_path): """Saves the certificate received from the ACME server. - :param certr: ACME "certificate" resource. - :type certr: :class:`acme.messages.Certificate` - - :param list chain_cert: + :param str cert_pem: + :param str chain_pem: :param str cert_path: Candidate path to a certificate. :param str chain_path: Candidate path to a certificate chain. :param str fullchain_path: Candidate path to a full cert chain. @@ -394,8 +366,6 @@ class Client(object): os.path.dirname(path), 0o755, os.geteuid(), self.config.strict_permissions) - cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) cert_file, abs_cert_path = _open_pem_file('cert_path', cert_path) @@ -406,20 +376,15 @@ class Client(object): logger.info("Server issued certificate; certificate written to %s", abs_cert_path) - if not chain_cert: - return abs_cert_path, None, None - else: - chain_pem = crypto_util.dump_pyopenssl_chain(chain_cert) + chain_file, abs_chain_path =\ + _open_pem_file('chain_path', chain_path) + fullchain_file, abs_fullchain_path =\ + _open_pem_file('fullchain_path', fullchain_path) - chain_file, abs_chain_path =\ - _open_pem_file('chain_path', chain_path) - fullchain_file, abs_fullchain_path =\ - _open_pem_file('fullchain_path', fullchain_path) + _save_chain(chain_pem, chain_file) + _save_chain(cert_pem + chain_pem, fullchain_file) - _save_chain(chain_pem, chain_file) - _save_chain(cert_pem + chain_pem, fullchain_file) - - return abs_cert_path, abs_chain_path, abs_fullchain_path + return abs_cert_path, abs_chain_path, abs_fullchain_path def deploy_certificate(self, domains, privkey_path, cert_path, chain_path, fullchain_path): diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 8368855cd..11721cc10 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,7 +14,6 @@ import six import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 -import josepy as jose from acme import crypto_util as acme_crypto_util @@ -367,16 +366,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """ # XXX: returns empty string when no chain is available, which # shuts up RenewableCert, but might not be the best solution... - - def _dump_cert(cert): - if isinstance(cert, jose.ComparableX509): - # pylint: disable=protected-access - cert = cert.wrapped - return OpenSSL.crypto.dump_certificate(filetype, cert) - - # assumes that OpenSSL.crypto.dump_certificate includes ending - # newline character - return b"".join(_dump_cert(cert) for cert in chain) + return acme_crypto_util.dump_pyopenssl_chain(chain, filetype) def notBefore(cert_path): @@ -443,3 +433,16 @@ def sha256sum(filename): with open(filename, 'rb') as f: sha256.update(f.read()) return sha256.hexdigest() + +def cert_and_chain_from_fullchain(fullchain_pem): + """Split fullchain_pem into cert_pem and chain_pem + + :param str fullchain_pem: concatenated cert + chain + + :returns: tuple of string cert_pem and chain_pem + :rtype: tuple + """ + cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, + OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + chain = fullchain_pem[len(cert):] + return (cert, chain) diff --git a/certbot/main.py b/certbot/main.py index d01f68920..eff4c9c8f 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1064,13 +1064,13 @@ def _csr_get_and_save_cert(config, le_client): """ csr, _ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(csr) + cert, chain = le_client.obtain_certificate_from_csr(csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) return None, None cert_path, _, fullchain_path = le_client.save_certificate( - certr, chain, config.cert_path, config.chain_path, config.fullchain_path) + cert, chain, config.cert_path, config.chain_path, config.fullchain_path) return cert_path, fullchain_path def renew_cert(config, plugins, lineage): diff --git a/certbot/renewal.py b/certbot/renewal.py index 024a815cc..ea5d87a5e 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -294,15 +294,12 @@ def renew_cert(config, domains, le_client, lineage): _avoid_invalidating_lineage(config, lineage, original_server) if not domains: domains = lineage.names() - new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) + new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert)) else: prior_version = lineage.latest_common_version() - new_cert = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped) - new_chain = crypto_util.dump_pyopenssl_chain(new_chain) # TODO: Check return value of save_successor lineage.save_successor(prior_version, new_cert, new_key.pem, new_chain, config) lineage.update_all_links_to(lineage.latest_common_version()) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a65341692..ed9c140e7 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,12 +4,8 @@ import shutil import tempfile import unittest -import josepy as jose -import OpenSSL import mock -from acme import errors as acme_errors - from certbot import account from certbot import errors from certbot import util @@ -134,7 +130,10 @@ class ClientTest(ClientTestCommon): self.config.allow_subset_of_names = False self.config.dry_run = False self.eg_domains = ["example.com", "www.example.com"] - self.eg_order = mock.MagicMock(authorizations=[None]) + self.eg_order = mock.MagicMock( + authorizations=[None], + fullchain_pem=mock.sentinel.fullchain_pem, + csr_pem=mock.sentinel.csr_pem) def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] @@ -143,9 +142,9 @@ class ClientTest(ClientTestCommon): def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() self.client.auth_handler.handle_authorizations.return_value = [None] - self.acme.request_issuance.return_value = mock.sentinel.certr - self.acme.fetch_chain.return_value = mock.sentinel.chain + self.acme.finalize_order.return_value = self.eg_order self.acme.new_order.return_value = self.eg_order + self.eg_order.update.return_value = self.eg_order def _check_obtain_certificate(self, auth_count=1): if auth_count == 1: @@ -155,27 +154,24 @@ class ClientTest(ClientTestCommon): else: self.assertEqual(self.client.auth_handler.handle_authorizations.call_count, auth_count) - authzr = self.client.auth_handler.handle_authorizations() - - self.acme.request_issuance.assert_called_once_with( - jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, CSR_SAN)), - authzr) - - self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) + self.acme.finalize_order.assert_called_once_with( + self.eg_order, mock.ANY) + @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.logger") @test_util.patch_get_utility() def test_obtain_certificate_from_csr(self, unused_mock_get_utility, - mock_logger): + mock_logger, mock_crypto_util): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) orderr = self.acme.new_order(test_csr.data) auth_handler.handle_authorizations(orderr, False) self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, orderr=orderr)) @@ -184,7 +180,7 @@ class ClientTest(ClientTestCommon): # Test for orderr=None self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, orderr=None)) @@ -198,41 +194,13 @@ class ClientTest(ClientTestCommon): test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_succeeded( - self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = [acme_errors.Error, - mock.sentinel.chain] - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - - orderr = self.acme.new_order(test_csr.data) - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - test_csr, - orderr=orderr)) - self.assertEqual(1, mock_get_utility().notification.call_count) - - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_failed(self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = acme_errors.Error - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - - orderr = self.acme.new_order(test_csr.data) - self.assertRaises( - acme_errors.Error, - self.client.obtain_certificate_from_csr, - test_csr, - orderr=orderr) - self.assertEqual(1, mock_get_utility().notification.call_count) - @mock.patch("certbot.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): csr = util.CSR(form="pem", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) self._test_obtain_certificate_common(mock.sentinel.key, csr) @@ -240,6 +208,8 @@ class ClientTest(ClientTestCommon): self.config.rsa_key_size, self.config.key_dir) mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) + mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with( + mock.sentinel.fullchain_pem) @mock.patch("certbot.client.crypto_util") @mock.patch("os.remove") @@ -248,6 +218,8 @@ class ClientTest(ClientTestCommon): key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = key + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) authzr = self._authzr_from_domains(["example.com"]) self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) @@ -255,6 +227,7 @@ class ClientTest(ClientTestCommon): self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) self.assertEqual(mock_crypto_util.init_save_csr.call_count, 2) self.assertEqual(mock_remove.call_count, 2) + self.assertEqual(mock_crypto_util.cert_and_chain_from_fullchain.call_count, 1) @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.acme_crypto_util") @@ -263,6 +236,8 @@ class ClientTest(ClientTestCommon): mock_acme_crypto.make_csr.return_value = CSR_SAN mock_crypto.make_key.return_value = mock.sentinel.key_pem key = util.Key(file=None, pem=mock.sentinel.key_pem) + mock_crypto.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) self.client.config.dry_run = True self._test_obtain_certificate_common(key, csr) @@ -272,6 +247,7 @@ class ClientTest(ClientTestCommon): mock.sentinel.key_pem, self.eg_domains, self.config.must_staple) mock_crypto.init_save_key.assert_not_called() mock_crypto.init_save_csr.assert_not_called() + self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1) def _authzr_from_domains(self, domains): authzr = [] @@ -294,7 +270,6 @@ class ClientTest(ClientTestCommon): authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr - self.eg_order.update().authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): @@ -302,13 +277,12 @@ class ClientTest(ClientTestCommon): self.assertEqual( result, - (mock.sentinel.certr, mock.sentinel.chain, key, csr)) + (mock.sentinel.cert, mock.sentinel.chain, key, csr)) self._check_obtain_certificate(auth_count) @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') - @mock.patch('OpenSSL.crypto.dump_certificate') - def test_obtain_and_enroll_certificate(self, mock_dump_certificate, + def test_obtain_and_enroll_certificate(self, mock_storage, mock_obtain_certificate): domains = ["example.com", "www.example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), @@ -324,7 +298,6 @@ class ClientTest(ClientTestCommon): self.assertFalse(self.client.obtain_and_enroll_certificate(domains, None)) self.assertTrue(mock_storage.call_count == 2) - self.assertTrue(mock_dump_certificate.call_count == 2) @mock.patch("certbot.cli.helpful_parser") def test_save_certificate(self, mock_parser): @@ -333,9 +306,8 @@ class ClientTest(ClientTestCommon): tmp_path = tempfile.mkdtemp() os.chmod(tmp_path, 0o755) # TODO: really?? - certr = mock.MagicMock(body=test_util.load_comparable_cert(certs[0])) - chain_cert = [test_util.load_comparable_cert(certs[0]), - test_util.load_comparable_cert(certs[1])] + cert_pem = test_util.load_vector(certs[0]) + chain_pem = (test_util.load_vector(certs[0]) + test_util.load_vector(certs[1])) candidate_cert_path = os.path.join(tmp_path, "certs", "cert_512.pem") candidate_chain_path = os.path.join(tmp_path, "chains", "chain.pem") candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") @@ -345,7 +317,7 @@ class ClientTest(ClientTestCommon): "--fullchain-path", candidate_fullchain_path] cert_path, chain_path, fullchain_path = self.client.save_certificate( - certr, chain_cert, candidate_cert_path, candidate_chain_path, + cert_pem, chain_pem, candidate_cert_path, candidate_chain_path, candidate_fullchain_path) self.assertEqual(os.path.dirname(cert_path), diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index f0e2c017e..00303fab3 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -373,5 +373,18 @@ class Sha256sumTest(unittest.TestCase): '914ffed8daf9e2c99d90ac95c77d54f32cbd556672facac380f0c063498df84e') +class CertAndChainFromFullchainTest(unittest.TestCase): + """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" + + def test_cert_and_chain_from_fullchain(self): + cert_pem = CERT + chain_pem = CERT + SS_CERT + fullchain_pem = cert_pem + chain_pem + from certbot.crypto_util import cert_and_chain_from_fullchain + cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem) + self.assertEqual(cert_out, cert_pem) + self.assertEqual(chain_out, chain_pem) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 60d8d6084..8434d11de 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -57,11 +57,6 @@ def load_cert(*names): return OpenSSL.crypto.load_certificate(loader, load_vector(*names)) -def load_comparable_cert(*names): - """Load ComparableX509 cert.""" - return jose.ComparableX509(load_cert(*names)) - - def load_csr(*names): """Load certificate request.""" loader = _guess_loader( From 990b211a76efb5a376cc4076bbbc2c2f0a2b3f2b Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 22 Feb 2018 12:33:55 -0800 Subject: [PATCH 29/56] Remove extra `:returns:` (#5611) --- certbot/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/client.py b/certbot/client.py index fc3848a5c..2d7288ce3 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -274,7 +274,7 @@ class Client(object): :param list domains: domains to get a certificate - :returns: :returns: certificate as PEM string, chain as PEM string, + :returns: certificate as PEM string, chain as PEM string, newly generated private key (`.util.Key`), and DER-encoded Certificate Signing Request (`.util.CSR`). :rtype: tuple From 1e46d26ac3511ca92e14c98c823a82286142ebb6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 22 Feb 2018 16:28:50 -0800 Subject: [PATCH 30/56] Fix ACMEv2 issues (#5612) * Add post wrapper to automatically add acme_version * Add uri to authzr. * Only add kid when account is set. * Add content_type when downloading certificate. * Only save new_authz URL when it exists. * Handle combinations in ACMEv1 and ACMEv2. * Add tests for ACMEv2 "combinations". --- acme/acme/client.py | 61 ++++++++++++-------- certbot/account.py | 17 ++++-- certbot/auth_handler.py | 12 +++- certbot/client.py | 4 +- certbot/tests/auth_handler_test.py | 93 ++++++++++++++++++++++++------ 5 files changed, 134 insertions(+), 53 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 97f529aae..9854aae31 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -70,7 +70,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes terms_of_service=terms_of_service) def _send_recv_regr(self, regr, body): - response = self.net.post(regr.uri, body, acme_version=self.acme_version) + response = self._post(regr.uri, body) # TODO: Boulder returns httplib.ACCEPTED #assert response.status_code == httplib.OK @@ -82,6 +82,13 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response, uri=regr.uri, terms_of_service=regr.terms_of_service) + def _post(self, *args, **kwargs): + """Wrapper around self.net.post that adds the acme_version. + + """ + kwargs.setdefault('acme_version', self.acme_version) + return self.net.post(*args, **kwargs) + def update_registration(self, regr, update=None): """Update registration. @@ -143,8 +150,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self.net.post(challb.uri, response, - acme_version=self.acme_version) + response = self._post(challb.uri, response) try: authzr_uri = response.links['up']['url'] except KeyError: @@ -216,12 +222,11 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .ClientError: If revocation is unsuccessful. """ - response = self.net.post(self.directory[messages.Revocation], + response = self._post(self.directory[messages.Revocation], messages.Revocation( certificate=cert, reason=rsn), - content_type=None, - acme_version=self.acme_version) + content_type=None) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') @@ -271,8 +276,7 @@ class Client(ClientBase): """ new_reg = messages.NewRegistration() if new_reg is None else new_reg - response = self.net.post(self.directory[new_reg], new_reg, - acme_version=1) + response = self._post(self.directory[new_reg], new_reg) # TODO: handle errors assert response.status_code == http_client.CREATED @@ -308,8 +312,7 @@ class Client(ClientBase): if new_authzr_uri is not None: logger.debug("request_challenges with new_authzr_uri deprecated.") new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz, new_authz, - acme_version=1) + response = self._post(self.directory.new_authz, new_authz) # TODO: handle errors assert response.status_code == http_client.CREATED return self._authzr_from_response(response, identifier) @@ -351,12 +354,11 @@ class Client(ClientBase): req = messages.CertificateRequest(csr=csr) content_type = DER_CONTENT_TYPE # TODO: add 'cert_type 'argument - response = self.net.post( + response = self._post( self.directory.new_cert, req, content_type=content_type, - headers={'Accept': content_type}, - acme_version=1) + headers={'Accept': content_type}) cert_chain_uri = response.links.get('up', {}).get('url') @@ -552,8 +554,7 @@ class ClientV2(ClientBase): :returns: Registration Resource. :rtype: `.RegistrationResource` """ - response = self.net.post(self.directory['newAccount'], new_account, - acme_version=2) + response = self._post(self.directory['newAccount'], new_account) # "Instance of 'Field' has no key/contact member" bug: # pylint: disable=no-member regr = self._regr_from_response(response) @@ -577,11 +578,11 @@ class ClientV2(ClientBase): identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=name)) order = messages.NewOrder(identifiers=identifiers) - response = self.net.post(self.directory['newOrder'], order) + response = self._post(self.directory['newOrder'], order) body = messages.Order.from_json(response.json()) authorizations = [] for url in body.authorizations: - authorizations.append(self._authzr_from_response(self.net.get(url))) + authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) return messages.OrderResource( body=body, uri=response.headers.get('Location'), @@ -643,7 +644,7 @@ class ClientV2(ClientBase): csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) - self.net.post(orderr.body.finalize, wrapped_csr) + self._post(orderr.body.finalize, wrapped_csr) while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) @@ -651,17 +652,29 @@ class ClientV2(ClientBase): if body.error is not None: raise errors.IssuanceError(body.error) if body.certificate is not None: - certificate_response = self.net.get(body.certificate).text + certificate_response = self.net.get(body.certificate, + content_type=DER_CONTENT_TYPE).text return orderr.update(body=body, fullchain_pem=certificate_response) raise errors.TimeoutError() class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but - supports V1 servers. + supports V1 servers. - :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint - :ivar .ClientBase client: either Client or ClientV2 + .. note:: While this class handles the majority of the differences + between versions of the ACME protocol, if you need to support an + ACME server based on version 3 or older of the IETF ACME draft + that uses combinations in authorizations (or lack thereof) to + signal that the client needs to complete something other than + any single challenge in the authorization to make it valid, the + user of this class needs to understand and handle these + differences themselves. This does not apply to either of Let's + Encrypt's endpoints where successfully completing any challenge + in an authorization will make it valid. + + :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint + :ivar .ClientBase client: either Client or ClientV2 """ def __init__(self, net, key, server): @@ -829,7 +842,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes } if acme_version == 2: kwargs["url"] = url - kwargs["kid"] = self.account["uri"] + # newAccount and revokeCert work without the kid + if self.account is not None: + kwargs["kid"] = self.account["uri"] kwargs["key"] = self.key # pylint: disable=star-args return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2) diff --git a/certbot/account.py b/certbot/account.py index 41e980097..70d9a7fc3 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -223,12 +223,17 @@ class AccountFileStorage(interfaces.AccountStorage): try: with open(self._regr_path(account_dir_path), "w") as regr_file: regr = account.regr - with_uri = RegistrationResourceWithNewAuthzrURI( - new_authzr_uri=acme.directory.new_authz, - body=regr.body, - uri=regr.uri, - terms_of_service=regr.terms_of_service) - regr_file.write(with_uri.json_dumps()) + # If we have a value for new-authz, save it for forwards + # compatibility with older versions of Certbot. If we don't + # have a value for new-authz, this is an ACMEv2 directory where + # an older version of Certbot won't work anyway. + if hasattr(acme.directory, "new-authz"): + regr = RegistrationResourceWithNewAuthzrURI( + new_authzr_uri=acme.directory.new_authz, + body=regr.body, + uri=regr.uri, + terms_of_service=regr.terms_of_service) + regr_file.write(regr.json_dumps()) if not regr_only: with util.safe_open(self._key_path(account_dir_path), "w", chmod=0o400) as key_file: diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 47d806b94..9cc10d4b4 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -24,7 +24,7 @@ class AuthHandler(object): :class:`~acme.challenges.Challenge` types :type auth: :class:`certbot.interfaces.IAuthenticator` - :ivar acme.client.Client acme: ACME client API. + :ivar acme.client.BackwardsCompatibleClientV2 acme: ACME client API. :ivar account: Client's Account :type account: :class:`certbot.account.Account` @@ -100,10 +100,16 @@ class AuthHandler(object): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") for dom in domains: + dom_challenges = self.authzr[dom].body.challenges + if self.acme.acme_version == 1: + combinations = self.authzr[dom].body.combinations + else: + combinations = tuple((i,) for i in range(len(dom_challenges))) + path = gen_challenge_path( - self.authzr[dom].body.challenges, + dom_challenges, self._get_chall_pref(dom), - self.authzr[dom].body.combinations) + combinations) dom_achalls = self._challenge_factory( dom, path) diff --git a/certbot/client.py b/certbot/client.py index 2d7288ce3..0f4fa760d 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -212,8 +212,8 @@ class Client(object): :ivar .IAuthenticator auth: Prepared (`.IAuthenticator.prepare`) authenticator that can solve ACME challenges. :ivar .IInstaller installer: Installer. - :ivar acme.client.Client acme: Optional ACME client API handle. - You might already have one from `register`. + :ivar acme.client.BackwardsCompatibleClientV2 acme: Optional ACME + client API handle. You might already have one from `register`. """ diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 3633b673d..394002206 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -81,6 +81,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_account = mock.Mock(key=util.Key("file_path", "PEM")) self.mock_net = mock.MagicMock(spec=acme_client.Client) + self.mock_net.acme_version = 1 self.handler = AuthHandler( self.mock_auth, self.mock_net, self.mock_account, []) @@ -90,13 +91,13 @@ class HandleAuthorizationsTest(unittest.TestCase): def tearDown(self): logging.disable(logging.NOTSET) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1(self, mock_poll): - mock_poll.side_effect = self._validate_all - - authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + def _test_name1_tls_sni_01_1_common(self, combos): + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos) mock_order = mock.MagicMock(authorizations=[authzr]) - authzr = self.handler.handle_authorizations(mock_order) + + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) @@ -112,8 +113,15 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) + def test_name1_tls_sni_01_1_acme_1(self): + self._test_name1_tls_sni_01_1_common(combos=True) + + def test_name1_tls_sni_01_1_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name1_tls_sni_01_1_common(combos=False) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll): + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self, mock_poll): mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) @@ -138,17 +146,43 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name3_tls_sni_01_3(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) - + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self, mock_poll): + self.mock_net.acme_version = 2 mock_poll.side_effect = self._validate_all + self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) + self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) + + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) + + self.assertEqual(self.mock_net.answer_challenge.call_count, 1) + + self.assertEqual(mock_poll.call_count, 1) + chall_update = mock_poll.call_args[0][0] + self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(len(chall_update.values()), 1) + + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + cleaned_up_achalls = self.mock_auth.cleanup.call_args[0][0] + self.assertEqual(len(cleaned_up_achalls), 1) + self.assertEqual(cleaned_up_achalls[0].typ, "tls-sni-01") + + # Length of authorizations list + self.assertEqual(len(authzr), 1) + + def _test_name3_tls_sni_01_3_common(self, combos): + self.mock_net.request_domain_challenges.side_effect = functools.partial( + gen_dom_authzr, challs=acme_util.CHALLENGES, combos=combos) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - authzr = self.handler.handle_authorizations(mock_order) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -167,6 +201,13 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 3) + def test_name3_tls_sni_01_3_common_acme_1(self): + self._test_name3_tls_sni_01_3_common(combos=True) + + def test_name3_tls_sni_01_3_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name3_tls_sni_01_3_common(combos=False) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_debug_challenges(self, mock_poll): zope.component.provideUtility( @@ -194,30 +235,44 @@ class HandleAuthorizationsTest(unittest.TestCase): mock_order = mock.MagicMock(authorizations=[]) self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_preferred_challenge_choice(self, mock_poll): - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + def _test_preferred_challenge_choice_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - self.handler.handle_authorizations(mock_order) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") - def test_preferred_challenges_not_supported(self): - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + def test_preferred_challenge_choice_common_acme_1(self): + self._test_preferred_challenge_choice_common(combos=True) + + def test_preferred_challenge_choice_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenge_choice_common(combos=False) + + def _test_preferred_challenges_not_supported_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] mock_order = mock.MagicMock(authorizations=authzrs) self.handler.pref_challs.append(challenges.HTTP01.typ) self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def test_preferred_challenges_not_supported_acme_1(self): + self._test_preferred_challenges_not_supported_common(combos=True) + + def test_preferred_challenges_not_supported_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenges_not_supported_common(combos=False) + def _validate_all(self, unused_1, unused_2): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] From f3a0deba840f1c6bc1510ee71a0e70035fa8488c Mon Sep 17 00:00:00 2001 From: Nick Bebout Date: Fri, 23 Feb 2018 15:26:11 -0600 Subject: [PATCH 31/56] Remove min version of setuptools (#5617) --- acme/setup.py | 4 +--- certbot-apache/setup.py | 4 +--- certbot-dns-cloudflare/setup.py | 4 +--- certbot-dns-cloudxns/setup.py | 4 +--- certbot-dns-digitalocean/setup.py | 4 +--- certbot-dns-dnsimple/setup.py | 4 +--- certbot-dns-dnsmadeeasy/setup.py | 4 +--- certbot-dns-google/setup.py | 4 +--- certbot-dns-luadns/setup.py | 4 +--- certbot-dns-nsone/setup.py | 4 +--- certbot-dns-rfc2136/setup.py | 4 +--- certbot-dns-route53/setup.py | 4 +--- certbot-nginx/setup.py | 4 +--- setup.py | 4 +--- 14 files changed, 14 insertions(+), 42 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index ce426cf74..51bbf0f71 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -19,9 +19,7 @@ install_requires = [ 'pyrfc3339', 'pytz', 'requests[security]>=2.4.1', # security extras added in 2.4.1 - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible ] diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 38f41e9f1..76d7f5ca5 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'mock', 'python-augeas', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 612e7259f..e1b84d1be 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'cloudflare>=1.5.1', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3157400c6..53ceb58ea 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1a68400fa..3330bdd67 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'mock', 'python-digitalocean>=1.11', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six', 'zope.interface', ] diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 35de47308..00a3c032a 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a946d00a4..36119ade0 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8585fc848..10fccd1ea 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -15,9 +15,7 @@ install_requires = [ 'mock', # for oauth2client.service_account.ServiceAccountCredentials 'oauth2client>=2.0', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', # already a dependency of google-api-python-client, but added for consistency 'httplib2' diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4fec37e29..b094e1818 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index dca9ebf27..e777d821b 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bfa72b50b..6fc6dca73 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dnspython', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8df687972..0fbeab31b 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -10,9 +10,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'boto3', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 152f77de8..a84efe2c3 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -13,9 +13,7 @@ install_requires = [ 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/setup.py b/setup.py index 47b5b0b2c..736ef467f 100644 --- a/setup.py +++ b/setup.py @@ -46,9 +46,7 @@ install_requires = [ 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] From 57bdc590dfcbe44da5565e8369997af633debadc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 26 Feb 2018 16:27:38 -0800 Subject: [PATCH 32/56] Add DNS Dockerfiles --- certbot-dns-cloudflare/Dockerfile | 5 +++++ certbot-dns-cloudxns/Dockerfile | 5 +++++ certbot-dns-digitalocean/Dockerfile | 5 +++++ certbot-dns-dnsimple/Dockerfile | 5 +++++ certbot-dns-dnsmadeeasy/Dockerfile | 5 +++++ certbot-dns-google/Dockerfile | 5 +++++ certbot-dns-luadns/Dockerfile | 5 +++++ certbot-dns-nsone/Dockerfile | 5 +++++ certbot-dns-rfc2136/Dockerfile | 5 +++++ certbot-dns-route53/Dockerfile | 5 +++++ 10 files changed, 50 insertions(+) create mode 100644 certbot-dns-cloudflare/Dockerfile create mode 100644 certbot-dns-cloudxns/Dockerfile create mode 100644 certbot-dns-digitalocean/Dockerfile create mode 100644 certbot-dns-dnsimple/Dockerfile create mode 100644 certbot-dns-dnsmadeeasy/Dockerfile create mode 100644 certbot-dns-google/Dockerfile create mode 100644 certbot-dns-luadns/Dockerfile create mode 100644 certbot-dns-nsone/Dockerfile create mode 100644 certbot-dns-rfc2136/Dockerfile create mode 100644 certbot-dns-route53/Dockerfile diff --git a/certbot-dns-cloudflare/Dockerfile b/certbot-dns-cloudflare/Dockerfile new file mode 100644 index 000000000..27dcc8751 --- /dev/null +++ b/certbot-dns-cloudflare/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-cloudflare + +RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare diff --git a/certbot-dns-cloudxns/Dockerfile b/certbot-dns-cloudxns/Dockerfile new file mode 100644 index 000000000..cc84ea65b --- /dev/null +++ b/certbot-dns-cloudxns/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-cloudxns + +RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns diff --git a/certbot-dns-digitalocean/Dockerfile b/certbot-dns-digitalocean/Dockerfile new file mode 100644 index 000000000..8bdd0619f --- /dev/null +++ b/certbot-dns-digitalocean/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-digitalocean + +RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean diff --git a/certbot-dns-dnsimple/Dockerfile b/certbot-dns-dnsimple/Dockerfile new file mode 100644 index 000000000..38d2be80e --- /dev/null +++ b/certbot-dns-dnsimple/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-dnsimple + +RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple diff --git a/certbot-dns-dnsmadeeasy/Dockerfile b/certbot-dns-dnsmadeeasy/Dockerfile new file mode 100644 index 000000000..ff7936925 --- /dev/null +++ b/certbot-dns-dnsmadeeasy/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-dnsmadeeasy + +RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy diff --git a/certbot-dns-google/Dockerfile b/certbot-dns-google/Dockerfile new file mode 100644 index 000000000..4a258d0ee --- /dev/null +++ b/certbot-dns-google/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-google + +RUN pip install --no-cache-dir --editable src/certbot-dns-google diff --git a/certbot-dns-luadns/Dockerfile b/certbot-dns-luadns/Dockerfile new file mode 100644 index 000000000..6efb4d777 --- /dev/null +++ b/certbot-dns-luadns/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-luadns + +RUN pip install --no-cache-dir --editable src/certbot-dns-luadns diff --git a/certbot-dns-nsone/Dockerfile b/certbot-dns-nsone/Dockerfile new file mode 100644 index 000000000..88fc13c57 --- /dev/null +++ b/certbot-dns-nsone/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-nsone + +RUN pip install --no-cache-dir --editable src/certbot-dns-nsone diff --git a/certbot-dns-rfc2136/Dockerfile b/certbot-dns-rfc2136/Dockerfile new file mode 100644 index 000000000..1b8feb2f8 --- /dev/null +++ b/certbot-dns-rfc2136/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-rfc2136 + +RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136 diff --git a/certbot-dns-route53/Dockerfile b/certbot-dns-route53/Dockerfile new file mode 100644 index 000000000..a1b8d6caf --- /dev/null +++ b/certbot-dns-route53/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-route53 + +RUN pip install --no-cache-dir --editable src/certbot-dns-route53 From 6f86267a26c9b748bd90113fe62157a9a455cdd2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 12:42:13 -0800 Subject: [PATCH 33/56] Fix revocation in ACMEv2 (#5626) * Allow revoke to pass in a url * Add revocation support to ACMEv2. * Provide regr for account based revocation. * Add revoke wrapper to BackwardsCompat client --- acme/acme/client.py | 52 +++++++++++++++++++++++++++++++++++----- acme/acme/client_test.py | 26 +++++++++++++++++--- certbot/main.py | 4 ++-- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 9854aae31..e3f6e845d 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -211,7 +211,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response, authzr.body.identifier, authzr.uri) return updated_authzr, response - def revoke(self, cert, rsn): + def _revoke(self, cert, rsn, url): """Revoke certificate. :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in @@ -219,14 +219,16 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :param int rsn: Reason code for certificate revocation. + :param str url: ACME URL to post to + :raises .ClientError: If revocation is unsuccessful. """ - response = self._post(self.directory[messages.Revocation], - messages.Revocation( - certificate=cert, - reason=rsn), - content_type=None) + response = self._post(url, + messages.Revocation( + certificate=cert, + reason=rsn), + content_type=None) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') @@ -528,6 +530,18 @@ class Client(ClientBase): "Recursion limit reached. Didn't get {0}".format(uri)) return chain + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self._revoke(cert, rsn, self.directory[messages.Revocation]) class ClientV2(ClientBase): @@ -657,6 +671,19 @@ class ClientV2(ClientBase): return orderr.update(body=body, fullchain_pem=certificate_response) raise errors.TimeoutError() + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self._revoke(cert, rsn, self.directory['revokeCert']) + class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but @@ -775,6 +802,19 @@ class BackwardsCompatibleClientV2(object): else: return self.client.finalize_order(orderr, deadline) + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self.client.revoke(cert, rsn) + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index acc5193ca..1e4db2884 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -40,6 +40,7 @@ DIRECTORY_V2 = messages.Directory({ 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce', 'newOrder': 'https://www.letsencrypt-demo.org/acme/new-order', + 'revokeCert': 'https://www.letsencrypt-demo.org/acme/revoke-cert', }) @@ -79,6 +80,9 @@ class ClientTestBase(unittest.TestCase): self.authzr = messages.AuthorizationResource( body=self.authz, uri=authzr_uri) + # Reason code for revocation + self.rsn = 1 + class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" @@ -251,6 +255,19 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.finalize_order(mock_orderr, mock_deadline) mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline) + def test_revoke(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" @@ -271,9 +288,6 @@ class ClientTest(ClientTestBase): uri='https://www.letsencrypt-demo.org/acme/cert/1', cert_chain_uri='https://www.letsencrypt-demo.org/ca') - # Reason code for revocation - self.rsn = 1 - from acme.client import Client self.client = Client( directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) @@ -752,6 +766,12 @@ class ClientV2Test(ClientTestBase): deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) self.assertRaises(errors.TimeoutError, self.client.finalize_order, self.orderr, deadline) + def test_revoke(self): + self.client.revoke(messages_test.CERT, self.rsn) + self.net.post.assert_called_once_with( + self.directory["revokeCert"], mock.ANY, content_type=None, + acme_version=2) + class MockJSONDeSerializable(jose.JSONDeSerializable): # pylint: disable=missing-docstring diff --git a/certbot/main.py b/certbot/main.py index 33c7730c6..7be852e83 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -982,11 +982,11 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config config.cert_path[0], config.key_path[0]) crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) key = jose.JWK.load(config.key_path[1]) + acme = client.acme_from_config_key(config, key) else: # revocation by account key logger.debug("Revoking %s using Account Key", config.cert_path[0]) acc, _ = _determine_account(config) - key = acc.key - acme = client.acme_from_config_key(config, key) + acme = client.acme_from_config_key(config, acc.key, acc.regr) cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] logger.debug("Reason code for revocation: %s", config.reason) From b18696b6a0967ddb609aeb5841b93e6214099080 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 16:47:43 -0800 Subject: [PATCH 34/56] Don't run tests with Python 2.6 (#5627) * Don't run tests with Python 2.6. * Revert "Don't run tests with Python 2.6." This reverts commit 4a9d778cca62ae2bec4cf060726e88f1fd66f374. * Revert changes to auto_test.py. --- letsencrypt-auto-source/tests/auto_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 8c2bfc079..d187452a1 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -287,8 +287,8 @@ class AutoTests(TestCase): self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: - self.assertIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertTrue('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This # conveniently sets us up to test the next 2 cases. @@ -296,8 +296,8 @@ class AutoTests(TestCase): # Test when neither phase-1 upgrade nor phase-2 upgrade is # needed (probably a common case): out, err = run_letsencrypt_auto() - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertNotIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertFalse('Creating virtual environment...' in out) def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" @@ -312,8 +312,8 @@ class AutoTests(TestCase): # Create venv saving the correct bootstrap script version out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: bootstrap_version = f.read() @@ -329,8 +329,8 @@ class AutoTests(TestCase): out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) def test_openssl_failure(self): """Make sure we stop if the openssl signature check fails.""" From a39d2fe55b760707718dcd8225b7dc42dcef4c9c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 18:05:33 -0800 Subject: [PATCH 35/56] Fix wildcard issuance (#5620) * Add is_wildcard_domain to certbot.util. * Error with --allow-subset-of-names and wildcards. * Fix issue preventing wildcard cert issuance. * Kill assumption domain is unique in auth_handler * fix typo and add test * update comments --- certbot/auth_handler.py | 158 ++++++++++++++++------------- certbot/cli.py | 5 + certbot/client.py | 7 +- certbot/tests/auth_handler_test.py | 80 +++++++-------- certbot/tests/cli_test.py | 4 + certbot/tests/client_test.py | 1 + certbot/tests/util_test.py | 20 ++++ certbot/util.py | 18 ++++ 8 files changed, 180 insertions(+), 113 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 9cc10d4b4..2b38e4af5 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -1,4 +1,5 @@ """ACME AuthHandler.""" +import collections import logging import time @@ -17,6 +18,10 @@ from certbot import interfaces logger = logging.getLogger(__name__) +AnnotatedAuthzr = collections.namedtuple("AnnotatedAuthzr", ["authzr", "achalls"]) +"""Stores an authorization resource and its active annotated challenges.""" + + class AuthHandler(object): """ACME Authorization Handler for a client. @@ -29,10 +34,8 @@ class AuthHandler(object): :ivar account: Client's Account :type account: :class:`certbot.account.Account` - :ivar dict authzr: ACME Authorization Resource dict where keys are domains - and values are :class:`acme.messages.AuthorizationResource` - :ivar list achalls: DV challenges in the form of - :class:`certbot.achallenges.AnnotatedChallenge` + :ivar aauthzrs: ACME Authorization Resources and their active challenges + :type aauthzrs: `list` of `AnnotatedAuthzr` :ivar list pref_challs: sorted user specified preferred challenges type strings with the most preferred challenge listed first @@ -42,12 +45,9 @@ class AuthHandler(object): self.acme = acme self.account = account - self.authzr = dict() + self.aauthzrs = [] self.pref_challs = pref_challs - # List must be used to keep responses straight. - self.achalls = [] - def handle_authorizations(self, orderr, best_effort=False): """Retrieve all authorizations for challenges. @@ -63,17 +63,15 @@ class AuthHandler(object): authorizations """ - authzrs = orderr.authorizations - for authzr in authzrs: - self.authzr[authzr.body.identifier.value] = authzr - domains = self.authzr.keys() + for authzr in orderr.authorizations: + self.aauthzrs.append(AnnotatedAuthzr(authzr, [])) - self._choose_challenges(domains) + self._choose_challenges() config = zope.component.getUtility(interfaces.IConfig) notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... - while self.achalls: + while self._has_challenges(): resp = self._solve_challenges() logger.info("Waiting for verification...") if config.debug_challenges: @@ -87,8 +85,8 @@ class AuthHandler(object): self.verify_authzr_complete() # Only return valid authorizations - retVal = [authzr for authzr in self.authzr.values() - if authzr.body.status == messages.STATUS_VALID] + retVal = [aauthzr.authzr for aauthzr in self.aauthzrs + if aauthzr.authzr.body.status == messages.STATUS_VALID] if not retVal: raise errors.AuthorizationError( @@ -96,41 +94,54 @@ class AuthHandler(object): return retVal - def _choose_challenges(self, domains): + def _choose_challenges(self): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") - for dom in domains: - dom_challenges = self.authzr[dom].body.challenges + for aauthzr in self.aauthzrs: + aauthzr_challenges = aauthzr.authzr.body.challenges if self.acme.acme_version == 1: - combinations = self.authzr[dom].body.combinations + combinations = aauthzr.authzr.body.combinations else: - combinations = tuple((i,) for i in range(len(dom_challenges))) + combinations = tuple((i,) for i in range(len(aauthzr_challenges))) path = gen_challenge_path( - dom_challenges, - self._get_chall_pref(dom), + aauthzr_challenges, + self._get_chall_pref(aauthzr.authzr.body.identifier.value), combinations) - dom_achalls = self._challenge_factory( - dom, path) - self.achalls.extend(dom_achalls) + aauthzr_achalls = self._challenge_factory( + aauthzr.authzr, path) + aauthzr.achalls.extend(aauthzr_achalls) + + def _has_challenges(self): + """Do we have any challenges to perform?""" + return any(aauthzr.achalls for aauthzr in self.aauthzrs) def _solve_challenges(self): """Get Responses for challenges from authenticators.""" resp = [] + all_achalls = self._get_all_achalls() with error_handler.ErrorHandler(self._cleanup_challenges): try: - if self.achalls: - resp = self.auth.perform(self.achalls) + if all_achalls: + resp = self.auth.perform(all_achalls) except errors.AuthorizationError: logger.critical("Failure in setting up challenges.") logger.info("Attempting to clean up outstanding challenges...") raise - assert len(resp) == len(self.achalls) + assert len(resp) == len(all_achalls) return resp + def _get_all_achalls(self): + """Return all active challenges.""" + all_achalls = [] + for aauthzr in self.aauthzrs: + all_achalls.extend(aauthzr.achalls) + + return all_achalls + def _respond(self, resp, best_effort): """Send/Receive confirmation of all challenges. @@ -139,69 +150,67 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 chall_update = dict() - active_achalls = self._send_responses(self.achalls, - resp, chall_update) + active_achalls = self._send_responses(resp, chall_update) # Check for updated status... try: self._poll_challenges(chall_update, best_effort) finally: - # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) - def _send_responses(self, achalls, resps, chall_update): + def _send_responses(self, resps, chall_update): """Send responses and make sure errors are handled. :param dict chall_update: parameter that is updated to hold - authzr -> list of outstanding solved annotated challenges + aauthzr index to list of outstanding solved annotated challenges """ active_achalls = [] - for achall, resp in six.moves.zip(achalls, resps): - # This line needs to be outside of the if block below to - # ensure failed challenges are cleaned up correctly - active_achalls.append(achall) + resps_iter = iter(resps) + for i, aauthzr in enumerate(self.aauthzrs): + for achall in aauthzr.achalls: + # This line needs to be outside of the if block below to + # ensure failed challenges are cleaned up correctly + active_achalls.append(achall) - # Don't send challenges for None and False authenticator responses - if resp is not None and resp: - self.acme.answer_challenge(achall.challb, resp) - # TODO: answer_challenge returns challr, with URI, - # that can be used in _find_updated_challr - # comparisons... - if achall.domain in chall_update: - chall_update[achall.domain].append(achall) - else: - chall_update[achall.domain] = [achall] + resp = next(resps_iter) + # Don't send challenges for None and False authenticator responses + if resp: + self.acme.answer_challenge(achall.challb, resp) + # TODO: answer_challenge returns challr, with URI, + # that can be used in _find_updated_challr + # comparisons... + chall_update.setdefault(i, []).append(achall) return active_achalls def _poll_challenges( self, chall_update, best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" - dom_to_check = set(chall_update.keys()) - comp_domains = set() + indices_to_check = set(chall_update.keys()) + comp_indices = set() rounds = 0 - while dom_to_check and rounds < max_rounds: + while indices_to_check and rounds < max_rounds: # TODO: Use retry-after... time.sleep(min_sleep) all_failed_achalls = set() - for domain in dom_to_check: + for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( - domain, chall_update[domain]) + index, chall_update[index]) - if len(comp_achalls) == len(chall_update[domain]): - comp_domains.add(domain) + if len(comp_achalls) == len(chall_update[index]): + comp_indices.add(index) elif not failed_achalls: for achall, _ in comp_achalls: - chall_update[domain].remove(achall) + chall_update[index].remove(achall) # We failed some challenges... damage control else: if best_effort: - comp_domains.add(domain) + comp_indices.add(index) logger.warning( "Challenge failed for domain %s", - domain) + self.aauthzrs[index].authzr.body.identifier.value) else: all_failed_achalls.update( updated for _, updated in failed_achalls) @@ -210,24 +219,26 @@ class AuthHandler(object): _report_failed_challs(all_failed_achalls) raise errors.FailedChallenges(all_failed_achalls) - dom_to_check -= comp_domains - comp_domains.clear() + indices_to_check -= comp_indices + comp_indices.clear() rounds += 1 - def _handle_check(self, domain, achalls): + def _handle_check(self, index, achalls): """Returns tuple of ('completed', 'failed').""" completed = [] failed = [] - self.authzr[domain], _ = self.acme.poll(self.authzr[domain]) - if self.authzr[domain].body.status == messages.STATUS_VALID: + original_aauthzr = self.aauthzrs[index] + updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) + self.aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) + if updated_authzr.body.status == messages.STATUS_VALID: return achalls, [] # Note: if the whole authorization is invalid, the individual failed # challenges will be determined here... for achall in achalls: updated_achall = achall.update(challb=self._find_updated_challb( - self.authzr[domain], achall)) + updated_authzr, achall)) # This does nothing for challenges that have yet to be decided yet. if updated_achall.status == messages.STATUS_VALID: @@ -285,14 +296,17 @@ class AuthHandler(object): logger.info("Cleaning up challenges") if achall_list is None: - achalls = self.achalls + achalls = self._get_all_achalls() else: achalls = achall_list if achalls: self.auth.cleanup(achalls) for achall in achalls: - self.achalls.remove(achall) + for aauthzr in self.aauthzrs: + if achall in aauthzr.achalls: + aauthzr.achalls.remove(achall) + break def verify_authzr_complete(self): """Verifies that all authorizations have been decided. @@ -301,15 +315,16 @@ class AuthHandler(object): :rtype: bool """ - for authzr in self.authzr.values(): + for aauthzr in self.aauthzrs: + authzr = aauthzr.authzr if (authzr.body.status != messages.STATUS_VALID and authzr.body.status != messages.STATUS_INVALID): raise errors.AuthorizationError("Incomplete authorizations") - def _challenge_factory(self, domain, path): + def _challenge_factory(self, authzr, path): """Construct Namedtuple Challenges - :param str domain: domain of the enrollee + :param messages.AuthorizationResource authzr: authorization :param list path: List of indices from `challenges`. @@ -323,8 +338,9 @@ class AuthHandler(object): achalls = [] for index in path: - challb = self.authzr[domain].body.challenges[index] - achalls.append(challb_to_achall(challb, self.account.key, domain)) + challb = authzr.body.challenges[index] + achalls.append(challb_to_achall( + challb, self.account.key, authzr.body.identifier.value)) return achalls diff --git a/certbot/cli.py b/certbot/cli.py index 09dd71d13..1c2273c8a 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -599,6 +599,11 @@ class HelpfulArgumentParser(object): if parsed_args.validate_hooks: hooks.validate_hooks(parsed_args) + if parsed_args.allow_subset_of_names: + if any(util.is_wildcard_domain(d) for d in parsed_args.domains): + raise errors.Error("Using --allow-subset-of-names with a" + " wildcard domain is not supported.") + possible_deprecation_warning(parsed_args) return parsed_args diff --git a/certbot/client.py b/certbot/client.py index 0f4fa760d..81fc0b802 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -298,7 +298,12 @@ class Client(object): auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] - if successful_domains != domains: + # allow_subset_of_names is currently disabled for wildcard + # certificates. The reason for this and checking allow_subset_of_names + # below is because successful_domains == domains is never true if + # domains contains a wildcard because the ACME spec forbids identifiers + # in authzs from containing a wildcard character. + if self.config.allow_subset_of_names and successful_domains != domains: if not self.config.dry_run: os.remove(key.file) os.remove(csr.file) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 394002206..7650d2c95 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -29,32 +29,31 @@ class ChallengeFactoryTest(unittest.TestCase): # Account is mocked... self.handler = AuthHandler(None, None, mock.Mock(key="mock_key"), []) - self.dom = "test" - self.handler.authzr[self.dom] = acme_util.gen_authzr( - messages.STATUS_PENDING, self.dom, acme_util.CHALLENGES, + self.authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", acme_util.CHALLENGES, [messages.STATUS_PENDING] * 6, False) def test_all(self): achalls = self.handler._challenge_factory( - self.dom, range(0, len(acme_util.CHALLENGES))) + self.authzr, range(0, len(acme_util.CHALLENGES))) self.assertEqual( [achall.chall for achall in achalls], acme_util.CHALLENGES) def test_one_tls_sni(self): - achalls = self.handler._challenge_factory(self.dom, [1]) + achalls = self.handler._challenge_factory(self.authzr, [1]) self.assertEqual( [achall.chall for achall in achalls], [acme_util.TLSSNI01]) def test_unrecognized(self): - self.handler.authzr["failure.com"] = acme_util.gen_authzr( - messages.STATUS_PENDING, "failure.com", - [mock.Mock(chall="chall", typ="unrecognized")], - [messages.STATUS_PENDING]) + authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", + [mock.Mock(chall="chall", typ="unrecognized")], + [messages.STATUS_PENDING]) self.assertRaises( - errors.Error, self.handler._challenge_factory, "failure.com", [0]) + errors.Error, self.handler._challenge_factory, authzr, [0]) class HandleAuthorizationsTest(unittest.TestCase): @@ -103,7 +102,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -134,7 +133,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -160,7 +159,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -190,12 +189,12 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] self.assertEqual(len(list(six.iterkeys(chall_update))), 3) - self.assertTrue("0" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["0"]), 1) - self.assertTrue("1" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["1"]), 1) - self.assertTrue("2" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["2"]), 1) + self.assertTrue(0 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[0]), 1) + self.assertTrue(1 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[1]), 1) + self.assertTrue(2 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[2]), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -274,14 +273,15 @@ class HandleAuthorizationsTest(unittest.TestCase): self._test_preferred_challenges_not_supported_common(combos=False) def _validate_all(self, unused_1, unused_2): - for dom in six.iterkeys(self.handler.authzr): - azr = self.handler.authzr[dom] - self.handler.authzr[dom] = acme_util.gen_authzr( + for i, aauthzr in enumerate(self.handler.aauthzrs): + azr = aauthzr.authzr + updated_azr = acme_util.gen_authzr( messages.STATUS_VALID, - dom, + azr.body.identifier.value, [challb.chall for challb in azr.body.challenges], [messages.STATUS_VALID] * len(azr.body.challenges), azr.body.combinations) + self.handler.aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) class PollChallengesTest(unittest.TestCase): @@ -290,7 +290,7 @@ class PollChallengesTest(unittest.TestCase): def setUp(self): from certbot.auth_handler import challb_to_achall - from certbot.auth_handler import AuthHandler + from certbot.auth_handler import AuthHandler, AnnotatedAuthzr # Account and network are mocked... self.mock_net = mock.MagicMock() @@ -298,40 +298,38 @@ class PollChallengesTest(unittest.TestCase): None, self.mock_net, mock.Mock(key="mock_key"), []) self.doms = ["0", "1", "2"] - self.handler.authzr[self.doms[0]] = acme_util.gen_authzr( + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[0], [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False) - - self.handler.authzr[self.doms[1]] = acme_util.gen_authzr( + [messages.STATUS_PENDING] * 2, False), [])) + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) - - self.handler.authzr[self.doms[2]] = acme_util.gen_authzr( + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) self.chall_update = {} - for dom in self.doms: - self.chall_update[dom] = [ - challb_to_achall(challb, mock.Mock(key="dummy_key"), dom) - for challb in self.handler.authzr[dom].body.challenges] + for i, aauthzr in enumerate(self.handler.aauthzrs): + self.chall_update[i] = [ + challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) + for challb in aauthzr.authzr.body.challenges] @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid self.handler._poll_challenges(self.chall_update, False) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_VALID) + for aauthzr in self.handler.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) @mock.patch("certbot.auth_handler.time") def test_poll_challenges_failure_best_effort(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid self.handler._poll_challenges(self.chall_update, True) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_PENDING) + for aauthzr in self.handler.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) @mock.patch("certbot.auth_handler.time") @test_util.patch_get_utility() @@ -345,7 +343,7 @@ class PollChallengesTest(unittest.TestCase): def test_unable_to_find_challenge_status(self, unused_mock_time): from certbot.auth_handler import challb_to_achall self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.chall_update[self.doms[0]].append( + self.chall_update[0].append( challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index c5935d722..1bba6991a 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -426,6 +426,10 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods namespace = self.parse(["--no-delete-after-revoke"]) self.assertFalse(namespace.delete_after_revoke) + def test_allow_subset_with_wildcard(self): + self.assertRaises(errors.Error, self.parse, + "--allow-subset-of-names -d *.example.org".split()) + class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index ed9c140e7..b51275d9e 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -222,6 +222,7 @@ class ClientTest(ClientTestCommon): mock.sentinel.chain) authzr = self._authzr_from_domains(["example.com"]) + self.config.allow_subset_of_names = True self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 50d323ffd..0e280f3ab 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -487,6 +487,26 @@ class EnforceDomainSanityTest(unittest.TestCase): self._call('this.is.xn--ls8h.tld') +class IsWildcardDomainTest(unittest.TestCase): + """Tests for is_wildcard_domain.""" + + def setUp(self): + self.wildcard = u"*.example.org" + self.no_wildcard = u"example.org" + + def _call(self, domain): + from certbot.util import is_wildcard_domain + return is_wildcard_domain(domain) + + def test_no_wildcard(self): + self.assertFalse(self._call(self.no_wildcard)) + self.assertFalse(self._call(self.no_wildcard.encode())) + + def test_wildcard(self): + self.assertTrue(self._call(self.wildcard)) + self.assertTrue(self._call(self.wildcard.encode())) + + class OsInfoTest(unittest.TestCase): """Test OS / distribution detection""" diff --git a/certbot/util.py b/certbot/util.py index f47c5da9c..f7ce6a3bc 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -601,6 +601,24 @@ def enforce_domain_sanity(domain): return domain +def is_wildcard_domain(domain): + """"Is domain a wildcard domain? + + :param damain: domain to check + :type domain: `bytes` or `str` or `unicode` + + :returns: True if domain is a wildcard, otherwise, False + :rtype: bool + + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + + return domain.startswith(wildcard_marker) + + def get_strict_version(normalized): """Converts a normalized version to a strict version. From e9bc4a319b9989a300dd574466a15edd581ee3c4 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 28 Feb 2018 21:31:47 +0200 Subject: [PATCH 36/56] Apache plugin wildcard support for ACMEv2 (#5608) In `deploy_cert()` and `enhance()`, the user will be presented with a dialog to choose from the VirtualHosts that can be covered by the wildcard domain name. The (multiple) selection result will then be handled in a similar way that we previously handled a single VirtualHost that was returned by the `_find_best_vhost()`. Additionally the selected VirtualHosts are added to a dictionary that maps selections to a wildcard domain to be reused in the later `enhance()` call and not forcing the user to select the same VirtualHosts again. * Apache plugin wildcard support * Present dialog only once per domain, added tests * Raise exception if no VHosts selected for wildcard domain --- certbot-apache/certbot_apache/configurator.py | 141 +++++++++++++++++- certbot-apache/certbot_apache/display_ops.py | 48 +++++- certbot-apache/certbot_apache/obj.py | 13 ++ .../certbot_apache/tests/configurator_test.py | 100 +++++++++++++ .../certbot_apache/tests/display_ops_test.py | 30 ++++ certbot/tests/main_test.py | 1 - 6 files changed, 316 insertions(+), 17 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 4bb2cbebd..6377bb114 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -5,6 +5,7 @@ import logging import os import pkg_resources import re +import six import socket import time @@ -152,6 +153,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc = dict() # Outstanding challenges self._chall_out = set() + # List of vhosts configured per wildcard domain on this run. + # used by deploy_cert() and enhance() + self._wildcard_vhosts = dict() # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) @@ -262,6 +266,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.aug, self.conf("server-root"), self.conf("vhost-root"), self.version, configurator=self) + def _wildcard_domain(self, domain): + """ + Checks if domain is a wildcard domain + + :param str domain: Domain to check + + :returns: If the domain is wildcard domain + :rtype: bool + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + return domain.startswith(wildcard_marker) + def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): """Deploys certificate to specified virtual host. @@ -280,9 +299,112 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): a lack of directives """ - # Choose vhost before (possible) enabling of mod_ssl, to keep the - # vhost choice namespace similar with the pre-validation one. - vhost = self.choose_vhost(domain) + vhosts = self.choose_vhosts(domain) + for vhost in vhosts: + self._deploy_cert(vhost, cert_path, key_path, chain_path, fullchain_path) + + def choose_vhosts(self, domain, create_if_no_ssl=True): + """ + Finds VirtualHosts that can be used with the provided domain + + :param str domain: Domain name to match VirtualHosts to + :param bool create_if_no_ssl: If found VirtualHost doesn't have a HTTPS + counterpart, should one get created + + :returns: List of VirtualHosts or None + :rtype: `list` of :class:`~certbot_apache.obj.VirtualHost` + """ + + if self._wildcard_domain(domain): + if domain in self._wildcard_vhosts: + # Vhosts for a wildcard domain were already selected + return self._wildcard_vhosts[domain] + # Ask user which VHosts to support. + # Returned objects are guaranteed to be ssl vhosts + return self._choose_vhosts_wildcard(domain, create_if_no_ssl) + else: + return [self.choose_vhost(domain)] + + def _vhosts_for_wildcard(self, domain): + """ + Get VHost objects for every VirtualHost that the user wants to handle + with the wildcard certificate. + """ + + # Collect all vhosts that match the name + matched = set() + for vhost in self.vhosts: + for name in vhost.get_names(): + if self._in_wildcard_scope(name, domain): + matched.add(vhost) + + return list(matched) + + def _in_wildcard_scope(self, name, domain): + """ + Helper method for _vhosts_for_wildcard() that makes sure that the domain + is in the scope of wildcard domain. + + eg. in scope: domain = *.wild.card, name = 1.wild.card + not in scope: domain = *.wild.card, name = 1.2.wild.card + """ + if len(name.split(".")) == len(domain.split(".")): + return fnmatch.fnmatch(name, domain) + + + def _choose_vhosts_wildcard(self, domain, create_ssl=True): + """Prompts user to choose vhosts to install a wildcard certificate for""" + + # Get all vhosts that are covered by the wildcard domain + vhosts = self._vhosts_for_wildcard(domain) + + # Go through the vhosts, making sure that we cover all the names + # present, but preferring the SSL vhosts + filtered_vhosts = dict() + for vhost in vhosts: + for name in vhost.get_names(): + if vhost.ssl: + # Always prefer SSL vhosts + filtered_vhosts[name] = vhost + elif name not in filtered_vhosts and create_ssl: + # Add if not in list previously + filtered_vhosts[name] = vhost + + # Only unique VHost objects + dialog_input = set([vhost for vhost in filtered_vhosts.values()]) + + # Ask the user which of names to enable, expect list of names back + dialog_output = display_ops.select_vhost_multiple(list(dialog_input)) + + if not dialog_output: + logger.error( + "No vhost exists with servername or alias for domain %s. " + "No vhost was selected. Please specify ServerName or ServerAlias " + "in the Apache config.", + domain) + raise errors.PluginError("No vhost selected") + + # Make sure we create SSL vhosts for the ones that are HTTP only + # if requested. + return_vhosts = list() + for vhost in dialog_output: + if not vhost.ssl: + return_vhosts.append(self.make_vhost_ssl(vhost)) + else: + return_vhosts.append(vhost) + + self._wildcard_vhosts[domain] = return_vhosts + return return_vhosts + + + def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): + """ + Helper function for deploy_cert() that handles the actual deployment + this exists because we might want to do multiple deployments per + domain originally passed for deploy_cert(). This is especially true + with wildcard certificates + """ + # This is done first so that ssl module is enabled and cert_path, # cert_key... can all be parsed appropriately @@ -311,7 +433,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "Unable to find cert and/or key directives") - logger.info("Deploying Certificate for %s to VirtualHost %s", domain, vhost.filep) + logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) if self.version < (2, 4, 8) or (chain_path and not fullchain_path): # install SSLCertificateFile, SSLCertificateKeyFile, @@ -327,8 +449,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "version of Apache") else: if not fullchain_path: - raise errors.PluginError("Please provide the --fullchain-path\ - option pointing to your full chain file") + raise errors.PluginError("Please provide the --fullchain-path " + "option pointing to your full chain file") set_cert_path = fullchain_path self.aug.set(path["cert_path"][-1], fullchain_path) self.aug.set(path["cert_key"][-1], key_path) @@ -391,7 +513,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.error( "No vhost exists with servername or alias of %s. " "No vhost was selected. Please specify ServerName or ServerAlias " - "in the Apache config, or split vhosts into separate files.", + "in the Apache config.", target_name) raise errors.PluginError("No vhost selected") elif temp: @@ -1376,8 +1498,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): except KeyError: raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) + + vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) try: - func(self.choose_vhost(domain), options) + for vhost in vhosts: + func(vhost, options) except errors.PluginError: logger.warning("Failed %s for %s", enhancement, domain) raise diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index 9529c1ab3..097b84b96 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -13,10 +13,44 @@ import certbot.display.util as display_util logger = logging.getLogger(__name__) +def select_vhost_multiple(vhosts): + """Select multiple Vhosts to install the certificate for + + :param vhosts: Available Apache VirtualHosts + :type vhosts: :class:`list` of type `~obj.Vhost` + + :returns: List of VirtualHosts + :rtype: :class:`list`of type `~obj.Vhost` + """ + if not vhosts: + return list() + tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] + # Remove the extra newline from the last entry + if len(tags_list): + tags_list[-1] = tags_list[-1][:-1] + code, names = zope.component.getUtility(interfaces.IDisplay).checklist( + "Which VirtualHosts would you like to install the wildcard certificate for?", + tags=tags_list, force_interactive=True) + if code == display_util.OK: + return_vhosts = _reversemap_vhosts(names, vhosts) + return return_vhosts + return [] + +def _reversemap_vhosts(names, vhosts): + """Helper function for select_vhost_multiple for mapping string + representations back to actual vhost objects""" + return_vhosts = list() + + for selection in names: + for vhost in vhosts: + if vhost.display_repr().strip() == selection.strip(): + return_vhosts.append(vhost) + return return_vhosts + def select_vhost(domain, vhosts): """Select an appropriate Apache Vhost. - :param vhosts: Available Apache Virtual Hosts + :param vhosts: Available Apache VirtualHosts :type vhosts: :class:`list` of type `~obj.Vhost` :returns: VirtualHost or `None` @@ -25,13 +59,11 @@ def select_vhost(domain, vhosts): """ if not vhosts: return None - while True: - code, tag = _vhost_menu(domain, vhosts) - if code == display_util.OK: - return vhosts[tag] - else: - return None - + code, tag = _vhost_menu(domain, vhosts) + if code == display_util.OK: + return vhosts[tag] + else: + return None def _vhost_menu(domain, vhosts): """Select an appropriate Apache Vhost. diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index 1e3579858..fcf3bfe08 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -167,6 +167,19 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods active="Yes" if self.enabled else "No", modmacro="Yes" if self.modmacro else "No")) + def display_repr(self): + """Return a representation of VHost to be used in dialog""" + return ( + "File: {filename}\n" + "Addresses: {addrs}\n" + "Names: {names}\n" + "HTTPS: {https}\n".format( + filename=self.filep, + addrs=", ".join(str(addr) for addr in self.addrs), + names=", ".join(self.get_names()), + https="Yes" if self.ssl else "No")) + + def __eq__(self, other): if isinstance(other, self.__class__): return (self.filep == other.filep and self.path == other.path and diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 8f34d33d3..c9bf9a63f 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1337,6 +1337,106 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enable_mod, "whatever") + def test_wildcard_domain(self): + # pylint: disable=protected-access + cases = {u"*.example.org": True, b"*.x.example.org": True, + u"a.example.org": False, b"a.x.example.org": False} + for key in cases.keys(): + self.assertEqual(self.config._wildcard_domain(key), cases[key]) + + def test_choose_vhosts_wildcard(self): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[3]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=True) + # Check that the dialog was called with one vh: certbot.demo + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[3]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].name == "certbot.demo") + self.assertTrue(vhs[0].ssl) + + self.assertFalse(vhs[0] == self.vh_truth[3]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_no_ssl(self, mock_makessl): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[1]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=False) + self.assertFalse(mock_makessl.called) + self.assertEquals(vhs[0], self.vh_truth[1]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._vhosts_for_wildcard") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_already_ssl(self, mock_makessl, mock_vh_for_w): + # pylint: disable=protected-access + # Already SSL vhost + mock_vh_for_w.return_value = [self.vh_truth[7]] + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[7]] + vhs = self.config._choose_vhosts_wildcard("whatever", + create_ssl=True) + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[7]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + # Ensure that make_vhost_ssl was not called, vhost.ssl == true + self.assertFalse(mock_makessl.called) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].ssl) + self.assertEquals(vhs[0], self.vh_truth[7]) + + + def test_deploy_cert_wildcard(self): + # pylint: disable=protected-access + mock_choose_vhosts = mock.MagicMock() + mock_choose_vhosts.return_value = [self.vh_truth[7]] + self.config._choose_vhosts_wildcard = mock_choose_vhosts + mock_d = "certbot_apache.configurator.ApacheConfigurator._deploy_cert" + with mock.patch(mock_d) as mock_dep: + self.config.deploy_cert("*.wildcard.example.org", "/tmp/path", + "/tmp/path", "/tmp/path", "/tmp/path") + self.assertTrue(mock_dep.called) + self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(self.vh_truth[7], mock_dep.call_args_list[0][0][0]) + + @mock.patch("certbot_apache.display_ops.select_vhost_multiple") + def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog): + # pylint: disable=protected-access + mock_dialog.return_value = [] + self.assertRaises(errors.PluginError, + self.config.deploy_cert, + "*.wild.cat", "/tmp/path", "/tmp/path", + "/tmp/path", "/tmp/path") + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_after_install(self, mock_choose): + # pylint: disable=protected-access + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertFalse(mock_choose.called) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_no_install(self, mock_choose): + mock_choose.return_value = [self.vh_truth[3]] + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertTrue(mock_choose.called) + + class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" # pylint: disable=protected-access diff --git a/certbot-apache/certbot_apache/tests/display_ops_test.py b/certbot-apache/certbot_apache/tests/display_ops_test.py index e59d411bd..df5cdbac0 100644 --- a/certbot-apache/certbot_apache/tests/display_ops_test.py +++ b/certbot-apache/certbot_apache/tests/display_ops_test.py @@ -11,9 +11,39 @@ from certbot.tests import util as certbot_util from certbot_apache import obj +from certbot_apache.display_ops import select_vhost_multiple from certbot_apache.tests import util +class SelectVhostMultiTest(unittest.TestCase): + """Tests for certbot_apache.display_ops.select_vhost_multiple.""" + + def setUp(self): + self.base_dir = "/example_path" + self.vhosts = util.get_vh_truth( + self.base_dir, "debian_apache_2_4/multiple_vhosts") + + def test_select_no_input(self): + self.assertFalse(select_vhost_multiple([])) + + @certbot_util.patch_get_utility() + def test_select_correct(self, mock_util): + mock_util().checklist.return_value = ( + display_util.OK, [self.vhosts[3].display_repr(), + self.vhosts[2].display_repr()]) + vhs = select_vhost_multiple([self.vhosts[3], + self.vhosts[2], + self.vhosts[1]]) + self.assertTrue(self.vhosts[2] in vhs) + self.assertTrue(self.vhosts[3] in vhs) + self.assertFalse(self.vhosts[1] in vhs) + + @certbot_util.patch_get_utility() + def test_select_cancel(self, mock_util): + mock_util().checklist.return_value = (display_util.CANCEL, "whatever") + vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]]) + self.assertFalse(vhs) + class SelectVhostTest(unittest.TestCase): """Tests for certbot_apache.display_ops.select_vhost.""" diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index c31a3fb33..b778f05ea 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -940,7 +940,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertRaises(errors.ConfigurationError, self._call, ['-d', (('a' * 50) + '.') * 10]) - # Bare IP address (this is actually a different error message now) self.assertRaises(errors.ConfigurationError, self._call, From 78735fa2c3ebaee6c7aba02bbf939597a9075cbb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 28 Feb 2018 16:08:06 -0800 Subject: [PATCH 37/56] Suggest DNS authenticator when it's needed (#5638) --- certbot/auth_handler.py | 16 ++++++++++++---- certbot/tests/auth_handler_test.py | 6 ++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 2b38e4af5..67d36c8cc 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -433,7 +433,7 @@ def _find_smart_path(challbs, preferences, combinations): combo_total = 0 if not best_combo: - _report_no_chall_path() + _report_no_chall_path(challbs) return best_combo @@ -454,15 +454,23 @@ def _find_dumb_path(challbs, preferences): if supported: path.append(i) else: - _report_no_chall_path() + _report_no_chall_path(challbs) return path -def _report_no_chall_path(): - """Logs and raises an error that no satisfiable chall path exists.""" +def _report_no_chall_path(challbs): + """Logs and raises an error that no satisfiable chall path exists. + + :param challbs: challenges from the authorization that can't be satisfied + + """ msg = ("Client with the currently selected authenticator does not support " "any combination of challenges that will satisfy the CA.") + if len(challbs) == 1 and isinstance(challbs[0].chall, challenges.DNS01): + msg += ( + " You may need to use an authenticator " + "plugin that can do challenges over DNS.") logger.fatal(msg) raise errors.AuthorizationError(msg) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 7650d2c95..b6af3d0f5 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -272,6 +272,12 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_net.acme_version = 2 self._test_preferred_challenges_not_supported_common(combos=False) + def test_dns_only_challenge_not_supported(self): + authzrs = [gen_dom_authzr(domain="0", challs=[acme_util.DNS01])] + mock_order = mock.MagicMock(authorizations=authzrs) + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def _validate_all(self, unused_1, unused_2): for i, aauthzr in enumerate(self.handler.aauthzrs): azr = aauthzr.authzr From 38d5144fff79e10507ee7d53cb6664c8180d2245 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 08:25:32 -0800 Subject: [PATCH 38/56] Drop min coverage to 63 (#5641) --- tests/boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 24d224cb0..ea412b6b9 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -437,4 +437,4 @@ then . ./certbot-nginx/tests/boulder-integration.sh fi -coverage report --fail-under 64 -m +coverage report --fail-under 63 -m From 559220c2eff90975fb671945d780c2a757d0e167 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 10:11:15 -0800 Subject: [PATCH 39/56] Add basic ACMEv2 integration tests (#5635) * Use newer boulder config * Use ACMEv2 endpoint if requested * Add v2 integration tests * Work with unset variables * Add wildcard issuance test * quote domains --- .travis.yml | 6 +++++- tests/boulder-fetch.sh | 7 +++++++ tests/boulder-integration.sh | 6 ++++++ tests/integration/_common.sh | 5 +++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 42b8d679d..c62664180 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,11 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=1 + env: TOXENV=py27_install BOULDER_INTEGRATION=v1 + sudo: required + services: docker + - python: "2.7" + env: TOXENV=py27_install BOULDER_INTEGRATION=v2 sudo: required services: docker - python: "2.7" diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 08eb736c2..fc9cbaae7 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -16,6 +16,13 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml + +# If we're testing against ACMEv2, we need to use a newer boulder config for +# now. See https://github.com/letsencrypt/boulder#quickstart. +if [ "$BOULDER_INTEGRATION" = "v2" ]; then + sed -i 's/BOULDER_CONFIG_DIR: .*/BOULDER_CONFIG_DIR: test\/config-next/' docker-compose.yml +fi + docker-compose up -d set +x # reduce verbosity while waiting for boulder diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ea412b6b9..f2b0dcf60 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -430,6 +430,12 @@ for path in $archive $conf $live; do fi done +# Test ACMEv2-only features +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then + common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ + --manual-auth-hook ./tests/manual-dns-auth.sh +fi + # Most CI systems set this variable to true. # If the tests are running as part of CI, Nginx should be available. if ${CI:-false} || type nginx; diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index d151bdc3f..236090a14 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,6 +16,11 @@ certbot_test () { "$@" } +# Use local ACMEv2 endpoint if requested and SERVER isn't already set. +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" -a -z "${SERVER:+x}" ]; then + SERVER="http://localhost:4001/directory" +fi + certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" omit_patterns="$omit_patterns,*_test.py,*_test_*," From f0b337532cdc232add47c7bf98401eb7d75ca615 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 1 Mar 2018 14:05:50 -0800 Subject: [PATCH 40/56] Nginx plugin wildcard support for ACMEv2 (#5619) * support wildcards for deploy_cert * support wildcards for enhance * redirect enhance and some tests * update tests * add display_ops and display_repr * update display_ops_test and errors found * say server block * match redirects properly * functional code * start adding tests and lint errors * add configurator tests * lint * change message to be generic to installation and enhancement * remove _wildcard_domain * take selecting vhosts out of loop * remove extra newline * filter wildcard vhosts by port * lint * don't filter by domain * [^.]+ * lint * make vhost hashable * one more tuple --- certbot-nginx/certbot_nginx/configurator.py | 199 ++++++++++++++---- certbot-nginx/certbot_nginx/display_ops.py | 44 ++++ certbot-nginx/certbot_nginx/http_01.py | 6 +- certbot-nginx/certbot_nginx/obj.py | 17 ++ .../certbot_nginx/tests/configurator_test.py | 100 ++++++++- .../certbot_nginx/tests/display_ops_test.py | 45 ++++ .../certbot_nginx/tests/tls_sni_01_test.py | 4 +- certbot-nginx/certbot_nginx/tls_sni_01.py | 7 +- 8 files changed, 370 insertions(+), 52 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/display_ops.py create mode 100644 certbot-nginx/certbot_nginx/tests/display_ops_test.py diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 9f091c0fd..e4d87744e 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -23,6 +23,7 @@ from certbot import util from certbot.plugins import common from certbot_nginx import constants +from certbot_nginx import display_ops from certbot_nginx import nginxparser from certbot_nginx import parser from certbot_nginx import tls_sni_01 @@ -92,6 +93,11 @@ class NginxConfigurator(common.Installer): # For creating new vhosts if no names match self.new_vhost = None + # List of vhosts configured per wildcard domain on this run. + # used by deploy_cert() and enhance() + self._wildcard_vhosts = {} + self._wildcard_redirect_vhosts = {} + # Add number of outstanding challenges self._chall_out = 0 @@ -146,6 +152,7 @@ class NginxConfigurator(common.Installer): raise errors.PluginError( 'Unable to lock %s', self.conf('server-root')) + # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): @@ -166,14 +173,24 @@ class NginxConfigurator(common.Installer): "The nginx plugin currently requires --fullchain-path to " "install a cert.") - vhost = self.choose_vhost(domain, create_if_no_match=True) + vhosts = self.choose_vhosts(domain, create_if_no_match=True) + for vhost in vhosts: + self._deploy_cert(vhost, cert_path, key_path, chain_path, fullchain_path) + + def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): + # pylint: disable=unused-argument + """ + Helper function for deploy_cert() that handles the actual deployment + this exists because we might want to do multiple deployments per + domain originally passed for deploy_cert(). This is especially true + with wildcard certificates + """ cert_directives = [['\n ', 'ssl_certificate', ' ', fullchain_path], ['\n ', 'ssl_certificate_key', ' ', key_path]] self.parser.add_server_directives(vhost, cert_directives, replace=True) - logger.info("Deployed Certificate to VirtualHost %s for %s", - vhost.filep, ", ".join(vhost.names)) + logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, @@ -181,10 +198,61 @@ class NginxConfigurator(common.Installer): self.save_notes += "\tssl_certificate %s\n" % fullchain_path self.save_notes += "\tssl_certificate_key %s\n" % key_path + def _choose_vhosts_wildcard(self, domain, prefer_ssl, no_ssl_filter_port=None): + """Prompts user to choose vhosts to install a wildcard certificate for""" + if prefer_ssl: + vhosts_cache = self._wildcard_vhosts + preference_test = lambda x: x.ssl + else: + vhosts_cache = self._wildcard_redirect_vhosts + preference_test = lambda x: not x.ssl + + # Caching! + if domain in vhosts_cache: + # Vhosts for a wildcard domain were already selected + return vhosts_cache[domain] + + # Get all vhosts whether or not they are covered by the wildcard domain + vhosts = self.parser.get_vhosts() + + # Go through the vhosts, making sure that we cover all the names + # present, but preferring the SSL or non-SSL vhosts + filtered_vhosts = {} + for vhost in vhosts: + # Ensure we're listening non-sslishly on no_ssl_filter_port + if no_ssl_filter_port is not None: + if not self._vhost_listening_on_port_no_ssl(vhost, no_ssl_filter_port): + continue + for name in vhost.names: + if preference_test(vhost): + # Prefer either SSL or non-SSL vhosts + filtered_vhosts[name] = vhost + elif name not in filtered_vhosts: + # Add if not in list previously + filtered_vhosts[name] = vhost + + # Only unique VHost objects + dialog_input = set([vhost for vhost in filtered_vhosts.values()]) + + # Ask the user which of names to enable, expect list of names back + return_vhosts = display_ops.select_vhost_multiple(list(dialog_input)) + + for vhost in return_vhosts: + if domain not in vhosts_cache: + vhosts_cache[domain] = [] + vhosts_cache[domain].append(vhost) + + return return_vhosts + ####################### # Vhost parsing methods ####################### - def choose_vhost(self, target_name, create_if_no_match=False): + def _choose_vhost_single(self, target_name): + matches = self._get_ranked_matches(target_name) + vhosts = [x for x in [self._select_best_name_match(matches)] if x is not None] + return vhosts + + def choose_vhosts(self, target_name, create_if_no_match=False): """Chooses a virtual host based on the given domain name. .. note:: This makes the vhost SSL-enabled if it isn't already. Follows @@ -202,17 +270,19 @@ class NginxConfigurator(common.Installer): when there is no match found. If we can't choose a default, raise a MisconfigurationError. - :returns: ssl vhost associated with name - :rtype: :class:`~certbot_nginx.obj.VirtualHost` + :returns: ssl vhosts associated with name + :rtype: list of :class:`~certbot_nginx.obj.VirtualHost` """ - vhost = None - - matches = self._get_ranked_matches(target_name) - vhost = self._select_best_name_match(matches) - if not vhost: + if util.is_wildcard_domain(target_name): + # Ask user which VHosts to support. + vhosts = self._choose_vhosts_wildcard(target_name, prefer_ssl=True) + else: + vhosts = self._choose_vhost_single(target_name) + if not vhosts: if create_if_no_match: - vhost = self._vhost_from_duplicated_default(target_name) + # result will not be [None] because it errors on failure + vhosts = [self._vhost_from_duplicated_default(target_name)] else: # No matches. Raise a misconfiguration error. raise errors.MisconfigurationError( @@ -222,10 +292,11 @@ class NginxConfigurator(common.Installer): "nginx configuration: " "https://nginx.org/en/docs/http/server_names.html") % (target_name)) # Note: if we are enhancing with ocsp, vhost should already be ssl. - if not vhost.ssl: - self._make_server_ssl(vhost) + for vhost in vhosts: + if not vhost.ssl: + self._make_server_ssl(vhost) - return vhost + return vhosts def ipv6_info(self, port): """Returns tuple of booleans (ipv6_active, ipv6only_present) @@ -359,7 +430,7 @@ class NginxConfigurator(common.Installer): return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhost(self, target_name, port, create_if_no_match=False): + def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -377,15 +448,20 @@ class NginxConfigurator(common.Installer): when there is no match found. If we can't choose a default, raise a MisconfigurationError. - :returns: vhost associated with name - :rtype: :class:`~certbot_nginx.obj.VirtualHost` + :returns: vhosts associated with name + :rtype: list of :class:`~certbot_nginx.obj.VirtualHost` """ - matches = self._get_redirect_ranked_matches(target_name, port) - vhost = self._select_best_name_match(matches) - if not vhost and create_if_no_match: - vhost = self._vhost_from_duplicated_default(target_name, port=port) - return vhost + if util.is_wildcard_domain(target_name): + # Ask user which VHosts to enhance. + vhosts = self._choose_vhosts_wildcard(target_name, prefer_ssl=False, + no_ssl_filter_port=port) + else: + matches = self._get_redirect_ranked_matches(target_name, port) + vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] + if not vhosts and create_if_no_match: + vhosts = [self._vhost_from_duplicated_default(target_name, port=port)] + return vhosts def _port_matches(self, test_port, matching_port): # test_port is a number, matching is a number or "" or None @@ -395,6 +471,23 @@ class NginxConfigurator(common.Installer): else: return test_port == matching_port + def _vhost_listening_on_port_no_ssl(self, vhost, port): + found_matching_port = False + if len(vhost.addrs) == 0: + # if there are no listen directives at all, Nginx defaults to + # listening on port 80. + found_matching_port = (port == self.DEFAULT_LISTEN_PORT) + else: + for addr in vhost.addrs: + if self._port_matches(port, addr.get_port()) and addr.ssl == False: + found_matching_port = True + + if found_matching_port: + # make sure we don't have an 'ssl on' directive + return not self.parser.has_ssl_on_directive(vhost) + else: + return False + def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -411,21 +504,7 @@ class NginxConfigurator(common.Installer): all_vhosts = self.parser.get_vhosts() def _vhost_matches(vhost, port): - found_matching_port = False - if len(vhost.addrs) == 0: - # if there are no listen directives at all, Nginx defaults to - # listening on port 80. - found_matching_port = (port == self.DEFAULT_LISTEN_PORT) - else: - for addr in vhost.addrs: - if self._port_matches(port, addr.get_port()) and addr.ssl == False: - found_matching_port = True - - if found_matching_port: - # make sure we don't have an 'ssl on' directive - return not self.parser.has_ssl_on_directive(vhost) - else: - return False + return self._vhost_listening_on_port_no_ssl(vhost, port) matching_vhosts = [vhost for vhost in all_vhosts if _vhost_matches(vhost, port)] @@ -587,17 +666,31 @@ class NginxConfigurator(common.Installer): """ port = self.DEFAULT_LISTEN_PORT - vhost = None # If there are blocks listening plaintextishly on self.DEFAULT_LISTEN_PORT, # choose the most name-matching one. - vhost = self.choose_redirect_vhost(domain, port) + vhosts = self.choose_redirect_vhosts(domain, port) - if vhost is None: + if not vhosts: logger.info("No matching insecure server blocks listening on port %s found.", self.DEFAULT_LISTEN_PORT) return + for vhost in vhosts: + self._enable_redirect_single(domain, vhost) + + def _enable_redirect_single(self, domain, vhost): + """Redirect all equivalent HTTP traffic to ssl_vhost. + + If the vhost is listening plaintextishly, separate out the + relevant directives into a new server block and add a rewrite directive. + + .. note:: This function saves the configuration + + :param str domain: domain to enable redirect for + :param `~obj.Vhost` vhost: vhost to enable redirect for + """ + new_vhost = None if vhost.ssl: new_vhost = self.parser.duplicate_vhost(vhost, @@ -638,7 +731,18 @@ class NginxConfigurator(common.Installer): :type chain_path: `str` or `None` """ - vhost = self.choose_vhost(domain) + vhosts = self.choose_vhosts(domain) + for vhost in vhosts: + self._enable_ocsp_stapling_single(vhost, chain_path) + + def _enable_ocsp_stapling_single(self, vhost, chain_path): + """Include OCSP response in TLS handshake + + :param str vhost: vhost to enable OCSP response for + :param chain_path: chain file path + :type chain_path: `str` or `None` + + """ if self.version < (1, 3, 7): raise errors.PluginError("Version 1.3.7 or greater of nginx " "is needed to enable OCSP stapling") @@ -889,14 +993,23 @@ def _test_block_from_block(block): parser.comment_directive(test_block, 0) return test_block[:-1] + def _redirect_block_for_domain(domain): + updated_domain = domain + match_symbol = '=' + if util.is_wildcard_domain(domain): + match_symbol = '~' + updated_domain = updated_domain.replace('.', r'\.') + updated_domain = updated_domain.replace('*', '[^.]+') + updated_domain = '^' + updated_domain + '$' redirect_block = [[ - ['\n ', 'if', ' ', '($host', ' ', '=', ' ', '%s)' % domain, ' '], + ['\n ', 'if', ' ', '($host', ' ', match_symbol, ' ', '%s)' % updated_domain, ' '], [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], '\n ']], ['\n']] return redirect_block + def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. diff --git a/certbot-nginx/certbot_nginx/display_ops.py b/certbot-nginx/certbot_nginx/display_ops.py new file mode 100644 index 000000000..5d6bda6b0 --- /dev/null +++ b/certbot-nginx/certbot_nginx/display_ops.py @@ -0,0 +1,44 @@ +"""Contains UI methods for Nginx operations.""" +import logging + +import zope.component + +from certbot import interfaces + +import certbot.display.util as display_util + + +logger = logging.getLogger(__name__) + + +def select_vhost_multiple(vhosts): + """Select multiple Vhosts to install the certificate for + :param vhosts: Available Nginx VirtualHosts + :type vhosts: :class:`list` of type `~obj.Vhost` + :returns: List of VirtualHosts + :rtype: :class:`list`of type `~obj.Vhost` + """ + if not vhosts: + return list() + tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] + # Remove the extra newline from the last entry + if len(tags_list): + tags_list[-1] = tags_list[-1][:-1] + code, names = zope.component.getUtility(interfaces.IDisplay).checklist( + "Which server blocks would you like to modify?", + tags=tags_list, force_interactive=True) + if code == display_util.OK: + return_vhosts = _reversemap_vhosts(names, vhosts) + return return_vhosts + return [] + +def _reversemap_vhosts(names, vhosts): + """Helper function for select_vhost_multiple for mapping string + representations back to actual vhost objects""" + return_vhosts = list() + + for selection in names: + for vhost in vhosts: + if vhost.display_repr().strip() == selection.strip(): + return_vhosts.append(vhost) + return return_vhosts diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index c0dec061a..0b1b2bfe0 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -179,13 +179,17 @@ class NginxHttp01(common.ChallengePerformer): """ try: - vhost = self.configurator.choose_redirect_vhost(achall.domain, + vhosts = self.configurator.choose_redirect_vhosts(achall.domain, '%i' % self.configurator.config.http01_port, create_if_no_match=True) except errors.MisconfigurationError: # Couldn't find either a matching name+port server block # or a port+default_server block, so create a dummy block return self._make_server_block(achall) + # len is max 1 because Nginx doesn't authenticate wildcards + # if len were or vhosts None, we would have errored + vhost = vhosts[0] + # Modify existing server block validation = achall.validation(achall.account_key) validation_path = self._get_validation_path(achall) diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index e8dc8936d..3625a95b9 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -193,6 +193,11 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return False + def __hash__(self): + return hash((self.filep, tuple(self.path), + tuple(self.addrs), tuple(self.names), + self.ssl, self.enabled)) + def contains_list(self, test): """Determine if raw server block contains test list at top level """ @@ -216,3 +221,15 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods for a in self.addrs: if not a.ipv6: return True + + def display_repr(self): + """Return a representation of VHost to be used in dialog""" + return ( + "File: {filename}\n" + "Addresses: {addrs}\n" + "Names: {names}\n" + "HTTPS: {https}\n".format( + filename=self.filep, + addrs=", ".join(str(addr) for addr in self.addrs), + names=", ".join(self.names), + https="Yes" if self.ssl else "No")) diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index acb7ee282..722ba68bf 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -128,7 +128,7 @@ class NginxConfiguratorTest(util.NginxTest): ['#', parser.COMMENT]]]], parsed[0]) - def test_choose_vhost(self): + def test_choose_vhosts(self): localhost_conf = set(['localhost', r'~^(www\.)?(example|bar)\.']) server_conf = set(['somename', 'another.alias', 'alias']) example_conf = set(['.example.com', 'example.*']) @@ -159,7 +159,7 @@ class NginxConfiguratorTest(util.NginxTest): '69.255.225.155'] for name in results: - vhost = self.config.choose_vhost(name) + vhost = self.config.choose_vhosts(name)[0] path = os.path.relpath(vhost.filep, self.temp_dir) self.assertEqual(results[name], vhost.names) @@ -173,7 +173,7 @@ class NginxConfiguratorTest(util.NginxTest): for name in bad_results: self.assertRaises(errors.MisconfigurationError, - self.config.choose_vhost, name) + self.config.choose_vhosts, name) def test_ipv6only(self): # ipv6_info: (ipv6_active, ipv6only_present) @@ -702,6 +702,100 @@ class NginxConfiguratorTest(util.NginxTest): self.config.rollback_checkpoints() self.assertTrue(mock_parser_load.call_count == 3) + def test_choose_vhosts_wildcard(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_select_vhs.return_value = [vhost] + vhs = self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=True) + # Check that the dialog was called with migration.com + self.assertTrue(vhost in mock_select_vhs.call_args[0][0]) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertEqual(vhs[0], vhost) + + def test_choose_vhosts_wildcard_redirect(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_select_vhs.return_value = [vhost] + vhs = self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=False) + # Check that the dialog was called with migration.com + self.assertTrue(vhost in mock_select_vhs.call_args[0][0]) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertEqual(vhs[0], vhost) + + def test_deploy_cert_wildcard(self): + # pylint: disable=protected-access + mock_choose_vhosts = mock.MagicMock() + vhost = [x for x in self.config.parser.get_vhosts() + if 'geese.com' in x.names][0] + mock_choose_vhosts.return_value = [vhost] + self.config._choose_vhosts_wildcard = mock_choose_vhosts + mock_d = "certbot_nginx.configurator.NginxConfigurator._deploy_cert" + with mock.patch(mock_d) as mock_dep: + self.config.deploy_cert("*.com", "/tmp/path", + "/tmp/path", "/tmp/path", "/tmp/path") + self.assertTrue(mock_dep.called) + self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(vhost, mock_dep.call_args_list[0][0][0]) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog): + # pylint: disable=protected-access + mock_dialog.return_value = [] + self.assertRaises(errors.PluginError, + self.config.deploy_cert, + "*.wild.cat", "/tmp/path", "/tmp/path", + "/tmp/path", "/tmp/path") + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_ocsp_after_install(self, mock_dialog): + # pylint: disable=protected-access + vhost = [x for x in self.config.parser.get_vhosts() + if 'geese.com' in x.names][0] + self.config._wildcard_vhosts["*.com"] = [vhost] + self.config.enhance("*.com", "staple-ocsp", "example/chain.pem") + self.assertFalse(mock_dialog.called) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_redirect_or_ocsp_no_install(self, mock_dialog): + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_dialog.return_value = [vhost] + self.config.enhance("*.com", "staple-ocsp", "example/chain.pem") + self.assertTrue(mock_dialog.called) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_double_redirect(self, mock_dialog): + # pylint: disable=protected-access + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + self.config._wildcard_redirect_vhosts["*.com"] = [vhost] + self.config.enhance("*.com", "redirect") + self.assertFalse(mock_dialog.called) + + def test_choose_vhosts_wildcard_no_ssl_filter_port(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [] + self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=False, + no_ssl_filter_port='80') + # Check that the dialog was called with only port 80 vhosts + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 4) + + class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" diff --git a/certbot-nginx/certbot_nginx/tests/display_ops_test.py b/certbot-nginx/certbot_nginx/tests/display_ops_test.py new file mode 100644 index 000000000..e3c6fb66b --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/display_ops_test.py @@ -0,0 +1,45 @@ +"""Test certbot_apache.display_ops.""" +import unittest + +from certbot.display import util as display_util + +from certbot.tests import util as certbot_util + +from certbot_nginx import parser + +from certbot_nginx.display_ops import select_vhost_multiple +from certbot_nginx.tests import util + + +class SelectVhostMultiTest(util.NginxTest): + """Tests for certbot_nginx.display_ops.select_vhost_multiple.""" + + def setUp(self): + super(SelectVhostMultiTest, self).setUp() + nparser = parser.NginxParser(self.config_path) + self.vhosts = nparser.get_vhosts() + + def test_select_no_input(self): + self.assertFalse(select_vhost_multiple([])) + + @certbot_util.patch_get_utility() + def test_select_correct(self, mock_util): + mock_util().checklist.return_value = ( + display_util.OK, [self.vhosts[3].display_repr(), + self.vhosts[2].display_repr()]) + vhs = select_vhost_multiple([self.vhosts[3], + self.vhosts[2], + self.vhosts[1]]) + self.assertTrue(self.vhosts[2] in vhs) + self.assertTrue(self.vhosts[3] in vhs) + self.assertFalse(self.vhosts[1] in vhs) + + @certbot_util.patch_get_utility() + def test_select_cancel(self, mock_util): + mock_util().checklist.return_value = (display_util.CANCEL, "whatever") + vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]]) + self.assertFalse(vhs) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 61ee293fa..72b65911c 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -61,10 +61,10 @@ class TlsSniPerformTest(util.NginxTest): shutil.rmtree(self.work_dir) @mock.patch("certbot_nginx.configurator" - ".NginxConfigurator.choose_vhost") + ".NginxConfigurator.choose_vhosts") def test_perform(self, mock_choose): self.sni.add_chall(self.achalls[1]) - mock_choose.return_value = None + mock_choose.return_value = [] result = self.sni.perform() self.assertFalse(result is None) diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index eca198bfe..0fd37e0cb 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -55,10 +55,11 @@ class NginxTlsSni01(common.TLSSNI01): self.configurator.config.tls_sni_01_port) for achall in self.achalls: - vhost = self.configurator.choose_vhost(achall.domain, create_if_no_match=True) + vhosts = self.configurator.choose_vhosts(achall.domain, create_if_no_match=True) - if vhost is not None and vhost.addrs: - addresses.append(list(vhost.addrs)) + # len is max 1 because Nginx doesn't authenticate wildcards + if vhosts and vhosts[0].addrs: + addresses.append(list(vhosts[0].addrs)) else: if ipv6: # If IPv6 is active in Nginx configuration From 8121acf2c1fed5514fd0a31a62f19dfbb92b2bb0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 14:54:48 -0800 Subject: [PATCH 41/56] Add user friendly wildcard error for ACMEv1 (#5636) * add WildcardUnsupportedError * Add friendly unsupported wildcard error msg * correct documentation * add version specifier --- acme/acme/client.py | 14 ++++++++++++++ acme/acme/client_test.py | 7 +++++++ acme/acme/errors.py | 3 +++ certbot/client.py | 30 +++++++++++++++++++++++------- certbot/tests/client_test.py | 2 +- 5 files changed, 48 insertions(+), 8 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index e3f6e845d..c6e897692 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -310,9 +310,17 @@ class Client(ClientBase): :returns: Authorization Resource. :rtype: `.AuthorizationResource` + :raises errors.WildcardUnsupportedError: if a wildcard is requested + """ if new_authzr_uri is not None: logger.debug("request_challenges with new_authzr_uri deprecated.") + + if identifier.value.startswith("*"): + raise errors.WildcardUnsupportedError( + "Requesting an authorization for a wildcard name is" + " forbidden by this version of the ACME protocol.") + new_authz = messages.NewAuthorization(identifier=identifier) response = self._post(self.directory.new_authz, new_authz) # TODO: handle errors @@ -333,6 +341,8 @@ class Client(ClientBase): :returns: Authorization Resource. :rtype: `.AuthorizationResource` + :raises errors.WildcardUnsupportedError: if a wildcard is requested + """ return self.request_challenges(messages.Identifier( typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) @@ -752,6 +762,10 @@ class BackwardsCompatibleClientV2(object): :returns: The newly created order. :rtype: OrderResource + + :raises errors.WildcardUnsupportedError: if a wildcard domain is + requested but unsupported by the ACME version + """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1e4db2884..060338360 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -376,6 +376,13 @@ class ClientTest(ClientTestBase): errors.UnexpectedUpdate, self.client.request_challenges, self.identifier) + def test_request_challenges_wildcard(self): + wildcard_identifier = messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value='*.example.org') + self.assertRaises( + errors.WildcardUnsupportedError, self.client.request_challenges, + wildcard_identifier) + def test_request_domain_challenges(self): self.client.request_challenges = mock.MagicMock() self.assertEqual( diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 991335958..97fa73614 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -115,3 +115,6 @@ class ConflictError(ClientError): self.location = location super(ConflictError, self).__init__() + +class WildcardUnsupportedError(Error): + """Error for when a wildcard is requested but is unsupported by ACME CA.""" diff --git a/certbot/client.py b/certbot/client.py index 81fc0b802..50d2262c4 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -12,6 +12,7 @@ import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util +from acme import errors as acme_errors from acme import messages import certbot @@ -258,10 +259,7 @@ class Client(object): logger.debug("CSR: %s", csr) if orderr is None: - orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr) - orderr = orderr.update(authorizations=authzr) - authzr = orderr.authorizations + orderr = self._get_order_and_authorizations(csr.data, best_effort=False) deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) orderr = self.acme.finalize_order(orderr, deadline) @@ -292,9 +290,8 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) - orderr = orderr.update(authorizations=authzr) + orderr = self._get_order_and_authorizations(csr.data, self.config.allow_subset_of_names) + authzr = orderr.authorizations auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] @@ -313,6 +310,25 @@ class Client(object): return cert, chain, key, csr + def _get_order_and_authorizations(self, csr_pem, best_effort): + """Request a new order and complete its authorizations. + + :param str csr_pem: A CSR in PEM format. + :param bool best_effort: True if failing to complete all + authorizations should not raise an exception + + :returns: order resource containing its completed authorizations + :rtype: acme.messages.OrderResource + + """ + try: + orderr = self.acme.new_order(csr_pem) + except acme_errors.WildcardUnsupportedError: + raise errors.Error("The currently selected ACME CA endpoint does" + " not support issuing wildcard certificates.") + authzr = self.auth_handler.handle_authorizations(orderr, best_effort) + return orderr.update(authorizations=authzr) + # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): """Obtain and enroll certificate. diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index b51275d9e..34595e463 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -184,7 +184,7 @@ class ClientTest(ClientTestCommon): self.client.obtain_certificate_from_csr( test_csr, orderr=None)) - auth_handler.handle_authorizations.assert_called_with(self.eg_order) + auth_handler.handle_authorizations.assert_called_with(self.eg_order, False) # Test for no auth_handler self.client.auth_handler = None From d8a54dc444a842e02c919b5092ebd745f25339e5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 14:55:45 -0800 Subject: [PATCH 42/56] Remove leading *. from default cert name. (#5639) --- certbot/client.py | 9 ++++++++- certbot/tests/client_test.py | 6 ++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 50d2262c4..eddf93e4f 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -354,7 +354,14 @@ class Client(object): "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") - new_name = certname if certname else domains[0] + if certname: + new_name = certname + elif util.is_wildcard_domain(domains[0]): + # Don't make files and directories starting with *. + new_name = domains[0][2:] + else: + new_name = domains[0] + if self.config.dry_run: logger.debug("Dry run: Skipping creating new lineage for %s", new_name) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 34595e463..5d01b103a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -285,7 +285,7 @@ class ClientTest(ClientTestCommon): @mock.patch('certbot.storage.RenewableCert.new_lineage') def test_obtain_and_enroll_certificate(self, mock_storage, mock_obtain_certificate): - domains = ["example.com", "www.example.com"] + domains = ["*.example.com", "example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), None) @@ -293,12 +293,14 @@ class ClientTest(ClientTestCommon): self.assertTrue(self.client.obtain_and_enroll_certificate(domains, "example_cert")) self.assertTrue(self.client.obtain_and_enroll_certificate(domains, None)) + self.assertTrue(self.client.obtain_and_enroll_certificate(domains[1:], None)) self.client.config.dry_run = True self.assertFalse(self.client.obtain_and_enroll_certificate(domains, None)) - self.assertTrue(mock_storage.call_count == 2) + names = [call[0][0] for call in mock_storage.call_args_list] + self.assertEqual(names, ["example_cert", "example.com", "example.com"]) @mock.patch("certbot.cli.helpful_parser") def test_save_certificate(self, mock_parser): From 8bc9cd67f0e6b445ee38342c904b8622c7f98878 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 1 Mar 2018 15:08:53 -0800 Subject: [PATCH 43/56] Fix ipv6only detection (#5648) * Fix ipv6only detection * move str() to inside ipv6_info * add regression test * Update to choose_vhosts --- certbot-nginx/certbot_nginx/configurator.py | 3 +++ .../certbot_nginx/tests/configurator_test.py | 12 ++++++++++++ .../testdata/etc_nginx/sites-enabled/ipv6ssl.com | 2 ++ 3 files changed, 17 insertions(+) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index e4d87744e..83e308bac 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -311,6 +311,9 @@ class NginxConfigurator(common.Installer): configuration, and existence of ipv6only directive for specified port :rtype: tuple of type (bool, bool) """ + # port should be a string, but it's easy to mess up, so let's + # make sure it is one + port = str(port) vhosts = self.parser.get_vhosts() ipv6_active = False ipv6only_present = False diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 722ba68bf..bffaef5e4 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -181,6 +181,18 @@ class NginxConfiguratorTest(util.NginxTest): # Port 443 has ipv6only=on because of ipv6ssl.com vhost self.assertEquals((True, True), self.config.ipv6_info("443")) + def test_ipv6only_detection(self): + self.config.version = (1, 3, 1) + + self.config.deploy_cert( + "ipv6.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + + for addr in self.config.choose_vhosts("ipv6.com")[0].addrs: + self.assertFalse(addr.ipv6only) def test_more_info(self): self.assertTrue('nginx.conf' in self.config.more_info()) diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com index d8f7eff12..875a9ee1b 100644 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com @@ -1,5 +1,7 @@ server { listen 443 ssl; listen [::]:443 ssl ipv6only=on; + listen 5001 ssl; + listen [::]:5001 ssl ipv6only=on; server_name ipv6ssl.com; } From e1878593d5608f908eebb262fe2c9c7dfcab55a1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Mar 2018 07:27:44 -0800 Subject: [PATCH 44/56] Ensure fullchain_pem in the order is unicode/str (#5654) * Decode fullchain_pem in ACMEv1 * Convert back to bytes in Certbot * document bytes are returned --- acme/acme/client.py | 4 ++-- acme/acme/client_test.py | 4 ++-- acme/acme/crypto_util.py | 3 +++ certbot/client.py | 5 +++-- certbot/crypto_util.py | 3 ++- certbot/tests/client_test.py | 22 ++++++++++++---------- certbot/tests/crypto_util_test.py | 4 ++-- 7 files changed, 26 insertions(+), 19 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c6e897692..d52c82a5c 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -809,8 +809,8 @@ class BackwardsCompatibleClientV2(object): 'certificate, please rerun the command for a new one.') cert = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) - chain = crypto_util.dump_pyopenssl_chain(chain) + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped).decode() + chain = crypto_util.dump_pyopenssl_chain(chain).decode() return orderr.update(fullchain_pem=(cert + chain)) else: diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 060338360..a0c27e74f 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -99,10 +99,10 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): self.chain = [wrapped, wrapped] self.cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped) + OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped).decode() single_chain = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, loaded) + OpenSSL.crypto.FILETYPE_PEM, loaded).decode() self.chain_pem = single_chain + single_chain self.fullchain_pem = self.cert_pem + self.chain_pem diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 07b55ae33..2281196eb 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -287,6 +287,9 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in :class:`josepy.util.ComparableX509`). + :returns: certificate chain bundle + :rtype: bytes + """ # XXX: returns empty string when no chain is available, which # shuts up RenewableCert, but might not be the best solution... diff --git a/certbot/client.py b/certbot/client.py index eddf93e4f..2992c0cec 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -244,7 +244,7 @@ class Client(object): than `authkey`. :param acme.messages.OrderResource orderr: contains authzrs - :returns: certificate and chain as PEM strings + :returns: certificate and chain as PEM byte strings :rtype: tuple """ @@ -263,7 +263,8 @@ class Client(object): deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) orderr = self.acme.finalize_order(orderr, deadline) - return crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) + cert, chain = crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) + return cert.encode(), chain.encode() def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 11721cc10..37118c591 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -441,8 +441,9 @@ def cert_and_chain_from_fullchain(fullchain_pem): :returns: tuple of string cert_pem and chain_pem :rtype: tuple + """ cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)).decode() chain = fullchain_pem[len(cert):] return (cert, chain) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 5d01b103a..0f2c58161 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -132,7 +132,6 @@ class ClientTest(ClientTestCommon): self.eg_domains = ["example.com", "www.example.com"] self.eg_order = mock.MagicMock( authorizations=[None], - fullchain_pem=mock.sentinel.fullchain_pem, csr_pem=mock.sentinel.csr_pem) def test_init_acme_verify_ssl(self): @@ -165,8 +164,7 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) orderr = self.acme.new_order(test_csr.data) auth_handler.handle_authorizations(orderr, False) @@ -199,8 +197,7 @@ class ClientTest(ClientTestCommon): csr = util.CSR(form="pem", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key - mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) self._test_obtain_certificate_common(mock.sentinel.key, csr) @@ -209,7 +206,7 @@ class ClientTest(ClientTestCommon): mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with( - mock.sentinel.fullchain_pem) + self.eg_order.fullchain_pem) @mock.patch("certbot.client.crypto_util") @mock.patch("os.remove") @@ -218,8 +215,7 @@ class ClientTest(ClientTestCommon): key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = key - mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) authzr = self._authzr_from_domains(["example.com"]) self.config.allow_subset_of_names = True @@ -237,8 +233,7 @@ class ClientTest(ClientTestCommon): mock_acme_crypto.make_csr.return_value = CSR_SAN mock_crypto.make_key.return_value = mock.sentinel.key_pem key = util.Key(file=None, pem=mock.sentinel.key_pem) - mock_crypto.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto.cert_and_chain_from_fullchain) self.client.config.dry_run = True self._test_obtain_certificate_common(key, csr) @@ -250,6 +245,13 @@ class ClientTest(ClientTestCommon): mock_crypto.init_save_csr.assert_not_called() self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1) + def _set_mock_from_fullchain(self, mock_from_fullchain): + mock_cert = mock.Mock() + mock_cert.encode.return_value = mock.sentinel.cert + mock_chain = mock.Mock() + mock_chain.encode.return_value = mock.sentinel.chain + mock_from_fullchain.return_value = (mock_cert, mock_chain) + def _authzr_from_domains(self, domains): authzr = [] diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 00303fab3..480139378 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -377,8 +377,8 @@ class CertAndChainFromFullchainTest(unittest.TestCase): """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" def test_cert_and_chain_from_fullchain(self): - cert_pem = CERT - chain_pem = CERT + SS_CERT + cert_pem = CERT.decode() + chain_pem = cert_pem + SS_CERT.decode() fullchain_pem = cert_pem + chain_pem from certbot.crypto_util import cert_and_chain_from_fullchain cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem) From cc344bfd1e080ad8ae253e0b1073a5ba7879583d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Mar 2018 09:50:19 -0800 Subject: [PATCH 45/56] Break lockstep between our packages (#5655) Fixes #5490. There's a lot of possibilities discussed in #5490, but I'll try and explain what I actually did here as succinctly as I can. Unfortunately, there's a fair bit to explain. My goal was to break lockstep and give us tests to ensure the minimum specified versions are correct without taking the time now to refactor our whole test setup. To handle specifying each package's minimum acme/certbot version, I added a requirements file to each package. This won't actually be included in the shipped package (because it's not in the MANIFEST). After creating these files and modifying tools/pip_install.sh to use them, I created a separate tox env for most packages (I kept the DNS plugins together for convenience). The reason this is necessary is because we currently use a single environment for each plugin, but if we used this approach for these tests we'd hit issues due to different installed plugins requiring different versions of acme/certbot. There's a lot more discussion about this in #5490 if you're interested in this piece. I unfortunately wasted a lot of time trying to remove the boilerplate this approach causes in tox.ini, but to do this I think we need negations described at complex factor conditions which hasn't made it into a tox release yet. The biggest missing piece here is how to make sure the oldest versions that are currently pinned to master get updated. Currently, they'll stay pinned that way without manual intervention and won't be properly testing the oldest version. I think we should solve this during the larger test/repo refactoring after the release because the tests are using the correct values now and I don't see a simple way around the problem. Once this lands, I'm planning on updating the test-everything tests to do integration tests with the "oldest" versions here. * break lockstep between packages * Use per package requirements files * add local oldest requirements files * update tox.ini * work with dev0 versions * Install requirements in separate step. * don't error when we don't have requirements * install latest packages in editable mode * Update .travis.yml * Add reminder comments * move dev to requirements * request acme[dev] * Update pip_install documentation --- .travis.yml | 2 +- certbot-apache/local-oldest-requirements.txt | 2 + certbot-apache/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-cloudflare/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-cloudxns/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-digitalocean/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-dnsimple/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-dnsmadeeasy/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-google/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-luadns/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-nsone/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-rfc2136/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-route53/setup.py | 6 ++- certbot-nginx/local-oldest-requirements.txt | 2 + certbot-nginx/setup.py | 10 ++-- local-oldest-requirements.txt | 1 + setup.py | 4 +- tools/pip_install.sh | 34 +++++++++--- tox.ini | 53 +++++++++++++++++-- 29 files changed, 154 insertions(+), 50 deletions(-) create mode 100644 certbot-apache/local-oldest-requirements.txt create mode 100644 certbot-dns-cloudflare/local-oldest-requirements.txt create mode 100644 certbot-dns-cloudxns/local-oldest-requirements.txt create mode 100644 certbot-dns-digitalocean/local-oldest-requirements.txt create mode 100644 certbot-dns-dnsimple/local-oldest-requirements.txt create mode 100644 certbot-dns-dnsmadeeasy/local-oldest-requirements.txt create mode 100644 certbot-dns-google/local-oldest-requirements.txt create mode 100644 certbot-dns-luadns/local-oldest-requirements.txt create mode 100644 certbot-dns-nsone/local-oldest-requirements.txt create mode 100644 certbot-dns-rfc2136/local-oldest-requirements.txt create mode 100644 certbot-dns-route53/local-oldest-requirements.txt create mode 100644 certbot-nginx/local-oldest-requirements.txt create mode 100644 local-oldest-requirements.txt diff --git a/.travis.yml b/.travis.yml index c62664180..9ec2f724b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ matrix: - python: "2.7" env: TOXENV=lint - python: "2.7" - env: TOXENV=py27-oldest + env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest' sudo: required services: docker - python: "3.4" diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-apache/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 336233bd4..7608c0647 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'mock', 'python-augeas', 'setuptools', diff --git a/certbot-dns-cloudflare/local-oldest-requirements.txt b/certbot-dns-cloudflare/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-cloudflare/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index e5687a9f5..4ed8e796d 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'cloudflare>=1.5.1', 'mock', 'setuptools', diff --git a/certbot-dns-cloudxns/local-oldest-requirements.txt b/certbot-dns-cloudxns/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-cloudxns/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 0ef31a90c..7f973709c 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-digitalocean/local-oldest-requirements.txt b/certbot-dns-digitalocean/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-digitalocean/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 11c2aea24..0ce91e64e 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'mock', 'python-digitalocean>=1.11', 'setuptools', diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 414a058fa..d12b26d83 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 18d773347..856eaba0f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-google/local-oldest-requirements.txt b/certbot-dns-google/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-google/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index d5def1bf9..0dfff0402 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', # 1.5 is the first version that supports oauth2client>=2.0 'google-api-python-client>=1.5', 'mock', diff --git a/certbot-dns-luadns/local-oldest-requirements.txt b/certbot-dns-luadns/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-luadns/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 13fa742d5..b255691dc 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-nsone/local-oldest-requirements.txt b/certbot-dns-nsone/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-nsone/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 01c9579c1..68d8f6cdb 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-rfc2136/local-oldest-requirements.txt b/certbot-dns-rfc2136/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-rfc2136/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 64b126595..3d6b3799b 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dnspython', 'mock', 'setuptools', diff --git a/certbot-dns-route53/local-oldest-requirements.txt b/certbot-dns-route53/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-route53/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index e45343f79..ad20725b5 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -5,9 +5,11 @@ from setuptools import find_packages version = '0.22.0.dev0' +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'boto3', 'mock', 'setuptools', diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt new file mode 100644 index 000000000..65f5a758e --- /dev/null +++ b/certbot-nginx/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +-e acme[dev] +-e .[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 96f8b834d..bb71cf19a 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -6,10 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + # This plugin works with an older version of acme, but Certbot does not. + # 0.22.0 is specified here to work around + # https://github.com/pypa/pip/issues/988. + 'acme>0.21.1', + 'certbot>0.21.1', 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt new file mode 100644 index 000000000..2346300a3 --- /dev/null +++ b/local-oldest-requirements.txt @@ -0,0 +1 @@ +-e acme[dev] diff --git a/setup.py b/setup.py index 9ac1a7ee7..3667a6976 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,9 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme=={0}'.format(version), + # Remember to update local-oldest-requirements.txt when changing the + # minimum acme version. + 'acme>0.21.1', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. diff --git a/tools/pip_install.sh b/tools/pip_install.sh index d2aae4a43..b385c5482 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,18 +1,30 @@ #!/bin/bash -e # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt and -# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's -# requirements file and tools/dev_constraints.txt is used. The other file -# always takes precedence over tools/dev_constraints.txt. +# to 1, a combination of tools/oldest_constraints.txt, +# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the +# top level of the package's directory is used, otherwise, a combination of +# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# other file always takes precedence over tools/dev_constraints.txt. If +# CERTBOT_OLDEST is set, this script must be run with `-e ` and +# no other arguments. # get the root of the Certbot repo tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -dev_constraints="$tools_dir/dev_constraints.txt" -merge_reqs="$tools_dir/merge_requirements.py" +all_constraints=$(mktemp) test_constraints=$(mktemp) -trap "rm -f $test_constraints" EXIT +trap "rm -f $all_constraints $test_constraints" EXIT if [ "$CERTBOT_OLDEST" = 1 ]; then + if [ "$1" != "-e" -o "$#" -ne "2" ]; then + echo "When CERTBOT_OLDEST is set, this script must be run with a single -e argument." + exit 1 + fi + pkg_dir=$(echo $2 | cut -f1 -d\[) # remove any extras such as [dev] + requirements="$pkg_dir/local-oldest-requirements.txt" + # packages like acme don't have any local oldest requirements + if [ ! -f "$requirements" ]; then + unset requirements + fi cp "$tools_dir/oldest_constraints.txt" "$test_constraints" else repo_root=$(dirname "$tools_dir") @@ -20,7 +32,13 @@ else sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" fi +"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \ + "$test_constraints" > "$all_constraints" + set -x # install the requested packages using the pinned requirements as constraints -pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@" +if [ -n "$requirements" ]; then + pip install -q --constraint "$all_constraints" --requirement "$requirements" +fi +pip install -q --constraint "$all_constraints" "$@" diff --git a/tox.ini b/tox.ini index 971aa7631..049220bbb 100644 --- a/tox.ini +++ b/tox.ini @@ -14,10 +14,7 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -all_packages = - acme[dev] \ - .[dev] \ - certbot-apache \ +dns_packages = certbot-dns-cloudflare \ certbot-dns-cloudxns \ certbot-dns-digitalocean \ @@ -27,7 +24,12 @@ all_packages = certbot-dns-luadns \ certbot-dns-nsone \ certbot-dns-rfc2136 \ - certbot-dns-route53 \ + certbot-dns-route53 +all_packages = + acme[dev] \ + .[dev] \ + certbot-apache \ + {[base]dns_packages} \ certbot-nginx \ letshelp-certbot install_packages = @@ -70,6 +72,47 @@ setenv = passenv = {[testenv]passenv} +[testenv:py27-acme-oldest] +commands = + {[base]install_and_test} acme[dev] +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-apache-oldest] +commands = + {[base]install_and_test} certbot-apache +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-certbot-oldest] +commands = + {[base]install_and_test} .[dev] +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-dns-oldest] +commands = + {[base]install_and_test} {[base]dns_packages} +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-nginx-oldest] +commands = + {[base]install_and_test} certbot-nginx + python tests/lock_test.py +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + [testenv:py27_install] basepython = python2.7 commands = From 441625c6102126f2d63daa964ecac4073e583d0a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 5 Mar 2018 22:49:02 +0200 Subject: [PATCH 46/56] Allow Google DNS plugin to write multiple TXT record values (#5652) * Allow Google DNS plugin to write multiple TXT record values in same resourcerecord * Atomic updates * Split rrsets request --- .../certbot_dns_google/dns_google.py | 63 ++++++++++++++++++- .../certbot_dns_google/dns_google_test.py | 61 +++++++++++++++++- 2 files changed, 119 insertions(+), 5 deletions(-) diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index cea754c06..ab8bf20de 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -107,6 +107,15 @@ class _GoogleClient(object): zone_id = self._find_managed_zone_id(domain) + record_contents = self.get_existing_txt_rrset(zone_id, record_name) + add_records = record_contents[:] + + if "\""+record_content+"\"" in record_contents: + # The process was interrupted previously and validation token exists + return + + add_records.append(record_content) + data = { "kind": "dns#change", "additions": [ @@ -114,12 +123,24 @@ class _GoogleClient(object): "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": [record_content, ], + "rrdatas": add_records, "ttl": record_ttl, }, ], } + if record_contents: + # We need to remove old records in the same request + data["deletions"] = [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": record_name + ".", + "rrdatas": record_contents, + "ttl": record_ttl, + }, + ] + changes = self.dns.changes() # changes | pylint: disable=no-member try: @@ -154,6 +175,8 @@ class _GoogleClient(object): logger.warn('Error finding zone. Skipping cleanup.') return + record_contents = self.get_existing_txt_rrset(zone_id, record_name) + data = { "kind": "dns#change", "deletions": [ @@ -161,12 +184,26 @@ class _GoogleClient(object): "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": [record_content, ], + "rrdatas": record_contents, "ttl": record_ttl, }, ], } + # Remove the record being deleted from the list + readd_contents = [r for r in record_contents if r != "\"" + record_content + "\""] + if readd_contents: + # We need to remove old records in the same request + data["additions"] = [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": record_name + ".", + "rrdatas": readd_contents, + "ttl": record_ttl, + }, + ] + changes = self.dns.changes() # changes | pylint: disable=no-member try: @@ -175,6 +212,28 @@ class _GoogleClient(object): except googleapiclient_errors.Error as e: logger.warn('Encountered error deleting TXT record: %s', e) + def get_existing_txt_rrset(self, zone_id, record_name): + """ + Get existing TXT records from the RRset for the record name. + + :param str zone_id: The ID of the managed zone. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + + :returns: List of TXT record values + :rtype: `list` of `string` + + """ + rrs_request = self.dns.resourceRecordSets() # pylint: disable=no-member + request = rrs_request.list(managedZone=zone_id, project=self.project_id) + response = request.execute() + # Add dot as the API returns absolute domains + record_name += "." + if response: + for rr in response["rrsets"]: + if rr["name"] == record_name and rr["type"] == "TXT": + return rr["rrdatas"] + return [] + def _find_managed_zone_id(self, domain): """ Find the managed zone for a given domain. diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 53f84dd6e..3291b2c3a 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -74,10 +74,15 @@ class GoogleClientTest(unittest.TestCase): mock_mz = mock.MagicMock() mock_mz.list.return_value.execute.side_effect = zone_request_side_effect + mock_rrs = mock.MagicMock() + rrsets = {"rrsets": [{"name": "_acme-challenge.example.org.", "type": "TXT", + "rrdatas": ["\"example-txt-contents\""]}]} + mock_rrs.list.return_value.execute.return_value = rrsets mock_changes = mock.MagicMock() client.dns.managedZones = mock.MagicMock(return_value=mock_mz) client.dns.changes = mock.MagicMock(return_value=mock_changes) + client.dns.resourceRecordSets = mock.MagicMock(return_value=mock_rrs) return client, mock_changes @@ -137,6 +142,30 @@ class GoogleClientTest(unittest.TestCase): managedZone=self.zone, project=PROJECT_ID) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_delete_old(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + mock_rrs.return_value = ["sample-txt-contents"] + client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + self.assertTrue(changes.create.called) + self.assertTrue("sample-txt-contents" in + changes.create.call_args_list[0][1]["body"]["deletions"][0]["rrdatas"]) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_noop(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + client.add_txt_record(DOMAIN, "_acme-challenge.example.org", + "example-txt-contents", self.record_ttl) + self.assertFalse(changes.create.called) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) @@ -172,7 +201,12 @@ class GoogleClientTest(unittest.TestCase): def test_del_txt_record(self, unused_credential_mock): client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) - client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + mock_rrs.return_value = ["\"sample-txt-contents\"", + "\"example-txt-contents\""] + client.del_txt_record(DOMAIN, "_acme-challenge.example.org", + "example-txt-contents", self.record_ttl) expected_body = { "kind": "dns#change", @@ -180,8 +214,17 @@ class GoogleClientTest(unittest.TestCase): { "kind": "dns#resourceRecordSet", "type": "TXT", - "name": self.record_name + ".", - "rrdatas": [self.record_content, ], + "name": "_acme-challenge.example.org.", + "rrdatas": ["\"sample-txt-contents\"", "\"example-txt-contents\""], + "ttl": self.record_ttl, + }, + ], + "additions": [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": "_acme-challenge.example.org.", + "rrdatas": ["\"sample-txt-contents\"", ], "ttl": self.record_ttl, }, ], @@ -217,6 +260,18 @@ class GoogleClientTest(unittest.TestCase): client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + # Record name mocked in setUp + found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") + self.assertEquals(found, ["\"example-txt-contents\""]) + not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") + self.assertEquals(not_found, []) + def test_get_project_id(self): from certbot_dns_google.dns_google import _GoogleClient From fe682e779b82ab0dfd72342369df630495c26a20 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Sun, 4 Mar 2018 16:24:42 +0200 Subject: [PATCH 47/56] ACMEv2 support for Route53 plugin --- .../certbot_dns_route53/dns_route53.py | 26 +++++++-- .../certbot_dns_route53/dns_route53_test.py | 54 +++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index 67462e369..c0e8e5495 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -85,9 +85,29 @@ class Authenticator(dns_common.DNSAuthenticator): zones.sort(key=lambda z: len(z[0]), reverse=True) return zones[0][1] + def _get_validation_rrset(self, zone_id, validation_domain_name): + validation_domain_name += "." + records = self.r53.list_resource_record_sets(HostedZoneId=zone_id) + for record in records["ResourceRecordSets"]: + if record["Name"] == validation_domain_name and record["Type"] == "TXT": + return record["ResourceRecords"] + return [] + def _change_txt_record(self, action, validation_domain_name, validation): zone_id = self._find_zone_id_for_domain(validation_domain_name) + rrecords = self._get_validation_rrset(zone_id, validation_domain_name) + challenge = {"Value": '"{0}"'.format(validation)} + if action == "DELETE": + if len(rrecords) > 1: + # Need to update instead, as we're not deleting the rrset + action = "UPSERT" + # Remove the record being deleted from the list + rrecords = [rr for rr in rrecords if rr != challenge] + else: + if challenge not in rrecords: + rrecords.append(challenge) + response = self.r53.change_resource_record_sets( HostedZoneId=zone_id, ChangeBatch={ @@ -99,11 +119,7 @@ class Authenticator(dns_common.DNSAuthenticator): "Name": validation_domain_name, "Type": "TXT", "TTL": self.ttl, - "ResourceRecords": [ - # For some reason TXT records need to be - # manually quoted. - {"Value": '"{0}"'.format(validation)} - ], + "ResourceRecords": rrecords, } } ] diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index d5f1b2816..9aec05b6e 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -178,6 +178,9 @@ class ClientTest(unittest.TestCase): def test_change_txt_record(self): self.client._find_zone_id_for_domain = mock.MagicMock() + self.client._get_validation_rrset = mock.MagicMock( + return_value=[] + ) self.client.r53.change_resource_record_sets = mock.MagicMock( return_value={"ChangeInfo": {"Id": 1}}) @@ -186,6 +189,57 @@ class ClientTest(unittest.TestCase): call_count = self.client.r53.change_resource_record_sets.call_count self.assertEqual(call_count, 1) + def test_change_txt_record_multirecord(self): + self.client._find_zone_id_for_domain = mock.MagicMock() + self.client._get_validation_rrset = mock.MagicMock() + self.client._get_validation_rrset.return_value = [ + {"Value": "\"pre-existing-value\""}, + {"Value": "\"pre-existing-value-two\""}, + ] + self.client.r53.change_resource_record_sets = mock.MagicMock( + return_value={"ChangeInfo": {"Id": 1}}) + + self.client._change_txt_record("DELETE", DOMAIN, "pre-existing-value") + + call_count = self.client.r53.change_resource_record_sets.call_count + call_args = self.client.r53.change_resource_record_sets.call_args_list[0][1] + call_args_batch = call_args["ChangeBatch"]["Changes"][0] + self.assertEqual(call_args_batch["Action"], "UPSERT") + self.assertEqual( + call_args_batch["ResourceRecordSet"]["ResourceRecords"], + [{"Value": "\"pre-existing-value-two\""}]) + + self.assertEqual(call_count, 1) + + def test_get_validation_rrset(self): + self.client.r53.list_resource_record_sets = mock.MagicMock( + return_value={"ResourceRecordSets": [ + {"Name": "_acme-challenge.example.org.", + "Type": "TXT", + "ResourceRecords": [ + {"Value": "\"validation-token\""}, + {"Value": "\"another-validation-token\""}, + ], + }, + {"Name": "_acme-challenge.example.org.", + "Type": "NS", + "ResourceRecords": [ + {"Value": "ns1.example.com"}, + ], + } + ]}) + rrset = self.client._get_validation_rrset("zoneid", + "_acme-challenge.example.org") + self.assertEquals(len(rrset), 2) + self.assertTrue({"Value": "\"another-validation-token\""} in rrset) + + def test_get_validation_rrset_empty(self): + self.client.r53.list_resource_record_sets = mock.MagicMock( + return_value={"ResourceRecordSets": []}) + rrset = self.client._get_validation_rrset("zoneid", + "_acme-challenge.example.org") + self.assertEquals(rrset, []) + def test_wait_for_change(self): self.client.r53.get_change = mock.MagicMock( side_effect=[{"ChangeInfo": {"Status": "PENDING"}}, From 7bc45121a13537cceef4e4bf53d4738925d55511 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Mar 2018 18:36:03 -0800 Subject: [PATCH 48/56] Remove the need for route53:ListResourceRecordSets * add test_change_txt_record_delete --- .../certbot_dns_route53/dns_route53.py | 24 ++++----- .../certbot_dns_route53/dns_route53_test.py | 54 ++++++++----------- 2 files changed, 31 insertions(+), 47 deletions(-) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index c0e8e5495..08b1d03f0 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -1,4 +1,5 @@ """Certbot Route53 authenticator plugin.""" +import collections import logging import time @@ -33,6 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) self.r53 = boto3.client("route53") + self._resource_records = collections.defaultdict(list) def more_info(self): # pylint: disable=missing-docstring,no-self-use return "Solve a DNS01 challenge using AWS Route53" @@ -85,28 +87,22 @@ class Authenticator(dns_common.DNSAuthenticator): zones.sort(key=lambda z: len(z[0]), reverse=True) return zones[0][1] - def _get_validation_rrset(self, zone_id, validation_domain_name): - validation_domain_name += "." - records = self.r53.list_resource_record_sets(HostedZoneId=zone_id) - for record in records["ResourceRecordSets"]: - if record["Name"] == validation_domain_name and record["Type"] == "TXT": - return record["ResourceRecords"] - return [] - def _change_txt_record(self, action, validation_domain_name, validation): zone_id = self._find_zone_id_for_domain(validation_domain_name) - rrecords = self._get_validation_rrset(zone_id, validation_domain_name) + rrecords = self._resource_records[validation_domain_name] challenge = {"Value": '"{0}"'.format(validation)} if action == "DELETE": - if len(rrecords) > 1: + # Remove the record being deleted from the list of tracked records + rrecords.remove(challenge) + if rrecords: # Need to update instead, as we're not deleting the rrset action = "UPSERT" - # Remove the record being deleted from the list - rrecords = [rr for rr in rrecords if rr != challenge] + else: + # Create a new list containing the record to use with DELETE + rrecords = [challenge] else: - if challenge not in rrecords: - rrecords.append(challenge) + rrecords.append(challenge) response = self.r53.change_resource_record_sets( HostedZoneId=zone_id, diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index 9aec05b6e..7534e132c 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -178,9 +178,6 @@ class ClientTest(unittest.TestCase): def test_change_txt_record(self): self.client._find_zone_id_for_domain = mock.MagicMock() - self.client._get_validation_rrset = mock.MagicMock( - return_value=[] - ) self.client.r53.change_resource_record_sets = mock.MagicMock( return_value={"ChangeInfo": {"Id": 1}}) @@ -189,10 +186,30 @@ class ClientTest(unittest.TestCase): call_count = self.client.r53.change_resource_record_sets.call_count self.assertEqual(call_count, 1) + def test_change_txt_record_delete(self): + self.client._find_zone_id_for_domain = mock.MagicMock() + self.client.r53.change_resource_record_sets = mock.MagicMock( + return_value={"ChangeInfo": {"Id": 1}}) + + validation = "some-value" + validation_record = {"Value": '"{0}"'.format(validation)} + self.client._resource_records[DOMAIN] = [validation_record] + + self.client._change_txt_record("DELETE", DOMAIN, validation) + + call_count = self.client.r53.change_resource_record_sets.call_count + self.assertEqual(call_count, 1) + call_args = self.client.r53.change_resource_record_sets.call_args_list[0][1] + call_args_batch = call_args["ChangeBatch"]["Changes"][0] + self.assertEqual(call_args_batch["Action"], "DELETE") + self.assertEqual( + call_args_batch["ResourceRecordSet"]["ResourceRecords"], + [validation_record]) + def test_change_txt_record_multirecord(self): self.client._find_zone_id_for_domain = mock.MagicMock() self.client._get_validation_rrset = mock.MagicMock() - self.client._get_validation_rrset.return_value = [ + self.client._resource_records[DOMAIN] = [ {"Value": "\"pre-existing-value\""}, {"Value": "\"pre-existing-value-two\""}, ] @@ -211,35 +228,6 @@ class ClientTest(unittest.TestCase): self.assertEqual(call_count, 1) - def test_get_validation_rrset(self): - self.client.r53.list_resource_record_sets = mock.MagicMock( - return_value={"ResourceRecordSets": [ - {"Name": "_acme-challenge.example.org.", - "Type": "TXT", - "ResourceRecords": [ - {"Value": "\"validation-token\""}, - {"Value": "\"another-validation-token\""}, - ], - }, - {"Name": "_acme-challenge.example.org.", - "Type": "NS", - "ResourceRecords": [ - {"Value": "ns1.example.com"}, - ], - } - ]}) - rrset = self.client._get_validation_rrset("zoneid", - "_acme-challenge.example.org") - self.assertEquals(len(rrset), 2) - self.assertTrue({"Value": "\"another-validation-token\""} in rrset) - - def test_get_validation_rrset_empty(self): - self.client.r53.list_resource_record_sets = mock.MagicMock( - return_value={"ResourceRecordSets": []}) - rrset = self.client._get_validation_rrset("zoneid", - "_acme-challenge.example.org") - self.assertEquals(rrset, []) - def test_wait_for_change(self): self.client.r53.get_change = mock.MagicMock( side_effect=[{"ChangeInfo": {"Status": "PENDING"}}, From cee9ac586ea43d355ede8eac71f5d145902169a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Mar 2018 07:20:34 -0800 Subject: [PATCH 49/56] Don't report coverage on Apache during integration tests (#5669) * ignore Apache coverage * drop min coverage to 67 --- tests/boulder-integration.sh | 2 +- tests/integration/_common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index f2b0dcf60..b5a305016 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -443,4 +443,4 @@ then . ./certbot-nginx/tests/boulder-integration.sh fi -coverage report --fail-under 63 -m +coverage report --fail-under 67 -m diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 236090a14..a8d35ed89 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -23,7 +23,7 @@ fi certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" - omit_patterns="$omit_patterns,*_test.py,*_test_*," + omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*" omit_patterns="$omit_patterns,certbot-compatibility-test/*,certbot-dns*/" coverage run \ --append \ From d62c56f9c91a920a07f338bbc7aa53b7329624ac Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Mar 2018 07:21:01 -0800 Subject: [PATCH 50/56] Remove the assumption the domain is unique in the manual plugin (#5670) * use entire achall as key * Add manual cleanup hook * use manual cleanup hook --- certbot/plugins/manual.py | 4 ++-- certbot/plugins/manual_test.py | 6 +++--- tests/boulder-integration.sh | 4 +++- tests/manual-dns-cleanup.sh | 3 +++ 4 files changed, 11 insertions(+), 6 deletions(-) create mode 100755 tests/manual-dns-cleanup.sh diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 07371ad34..614449d34 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -189,7 +189,7 @@ when it receives a TLS ClientHello with the SNI extension set to os.environ.update(env) _, out = hooks.execute(self.conf('auth-hook')) env['CERTBOT_AUTH_OUTPUT'] = out.strip() - self.env[achall.domain] = env + self.env[achall] = env def _perform_achall_manually(self, achall): validation = achall.validation(achall.account_key) @@ -215,7 +215,7 @@ when it receives a TLS ClientHello with the SNI extension set to def cleanup(self, achalls): # pylint: disable=missing-docstring if self.conf('cleanup-hook'): for achall in achalls: - env = self.env.pop(achall.domain) + env = self.env.pop(achall) if 'CERTBOT_TOKEN' not in env: os.environ.pop('CERTBOT_TOKEN', None) os.environ.update(env) diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index ac528e81c..e5c22b377 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -93,10 +93,10 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.auth.perform(self.achalls), [achall.response(achall.account_key) for achall in self.achalls]) self.assertEqual( - self.auth.env[self.dns_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.dns_achall]['CERTBOT_AUTH_OUTPUT'], dns_expected) self.assertEqual( - self.auth.env[self.http_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.http_achall]['CERTBOT_AUTH_OUTPUT'], http_expected) # tls_sni_01 challenge must be perform()ed above before we can # get the cert_path and key_path. @@ -107,7 +107,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.auth.tls_sni_01.get_z_domain(self.tls_sni_achall), 'novalidation') self.assertEqual( - self.auth.env[self.tls_sni_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.tls_sni_achall]['CERTBOT_AUTH_OUTPUT'], tls_sni_expected) @test_util.patch_get_utility() diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index b5a305016..2b92476fd 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -233,6 +233,7 @@ certname="dns.le.wtf" common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ --cert-name $certname \ --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ --renew-hook 'echo deploy >> "$HOOK_TEST"' @@ -433,7 +434,8 @@ done # Test ACMEv2-only features if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ - --manual-auth-hook ./tests/manual-dns-auth.sh + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh fi # Most CI systems set this variable to true. diff --git a/tests/manual-dns-cleanup.sh b/tests/manual-dns-cleanup.sh new file mode 100755 index 000000000..0c5c56b17 --- /dev/null +++ b/tests/manual-dns-cleanup.sh @@ -0,0 +1,3 @@ +#!/bin/sh +curl -X POST 'http://localhost:8055/clear-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}" From 6357e051f4841df8af69a8abf5a4ba3dc8578c3c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Mar 2018 15:32:22 -0800 Subject: [PATCH 51/56] Fallback without dns.resourceRecordSets.list permission (#5678) * Add rrset list fallback * List dns.resourceRecordSets.list as required * Handle list failures differently for add and del * Quote record content * disable not-callable for iter_entry_points * List update permission --- .../certbot_dns_google/__init__.py | 2 ++ .../certbot_dns_google/dns_google.py | 29 ++++++++++++++----- .../certbot_dns_google/dns_google_test.py | 14 ++++++++- certbot/plugins/disco.py | 1 + 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py index 7349a7696..f19266737 100644 --- a/certbot-dns-google/certbot_dns_google/__init__.py +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -29,6 +29,8 @@ for an account with the following permissions: * ``dns.managedZones.list`` * ``dns.resourceRecordSets.create`` * ``dns.resourceRecordSets.delete`` +* ``dns.resourceRecordSets.list`` +* ``dns.resourceRecordSets.update`` Google provides instructions for `creating a service account `_ and diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index ab8bf20de..e2088b357 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -108,6 +108,8 @@ class _GoogleClient(object): zone_id = self._find_managed_zone_id(domain) record_contents = self.get_existing_txt_rrset(zone_id, record_name) + if record_contents is None: + record_contents = [] add_records = record_contents[:] if "\""+record_content+"\"" in record_contents: @@ -176,6 +178,8 @@ class _GoogleClient(object): return record_contents = self.get_existing_txt_rrset(zone_id, record_name) + if record_contents is None: + record_contents = ["\"" + record_content + "\""] data = { "kind": "dns#change", @@ -216,23 +220,32 @@ class _GoogleClient(object): """ Get existing TXT records from the RRset for the record name. + If an error occurs while requesting the record set, it is suppressed + and None is returned. + :param str zone_id: The ID of the managed zone. :param str record_name: The record name (typically beginning with '_acme-challenge.'). - :returns: List of TXT record values - :rtype: `list` of `string` + :returns: List of TXT record values or None + :rtype: `list` of `string` or `None` """ rrs_request = self.dns.resourceRecordSets() # pylint: disable=no-member request = rrs_request.list(managedZone=zone_id, project=self.project_id) - response = request.execute() # Add dot as the API returns absolute domains record_name += "." - if response: - for rr in response["rrsets"]: - if rr["name"] == record_name and rr["type"] == "TXT": - return rr["rrdatas"] - return [] + try: + response = request.execute() + except googleapiclient_errors.Error: + logger.info("Unable to list existing records. If you're " + "requesting a wildcard certificate, this might not work.") + logger.debug("Error was:", exc_info=True) + else: + if response: + for rr in response["rrsets"]: + if rr["name"] == record_name and rr["type"] == "TXT": + return rr["rrdatas"] + return None def _find_managed_zone_id(self, domain): """ diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 3291b2c3a..afab847cf 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -270,7 +270,19 @@ class GoogleClientTest(unittest.TestCase): found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") self.assertEquals(found, ["\"example-txt-contents\""]) not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") - self.assertEquals(not_found, []) + self.assertEquals(not_found, None) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing_fallback(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + mock_execute = client.dns.resourceRecordSets.return_value.list.return_value.execute + mock_execute.side_effect = API_ERROR + + rrset = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") + self.assertFalse(rrset) def test_get_project_id(self): from certbot_dns_google.dns_google import _GoogleClient diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 5a7e07ec0..062c11650 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -190,6 +190,7 @@ class PluginsRegistry(collections.Mapping): def find_all(cls): """Find plugins using setuptools entry points.""" plugins = {} + # pylint: disable=not-callable entry_points = itertools.chain( pkg_resources.iter_entry_points( constants.SETUPTOOLS_PLUGINS_ENTRY_POINT), From e0ae356aa35adf22d154113e06dd01409df93bba Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Mar 2018 09:10:47 -0800 Subject: [PATCH 52/56] Upgrade pipstrap to 1.5.1 (#5681) * upgrade pipstrap to 1.5.1 * build leauto --- letsencrypt-auto-source/letsencrypt-auto | 38 +++++++++------------- letsencrypt-auto-source/pieces/pipstrap.py | 38 +++++++++------------- 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9ff1c1386..f97dc078d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1216,7 +1216,7 @@ UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 +"""A small script that can act as a trust root for installing pip >=8 Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, @@ -1274,7 +1274,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -1287,14 +1287,11 @@ maybe_argparse = ( if version_info < (2, 7, 0) else []) -# Pip has no dependencies, as it vendors everything: -PIP_PACKAGE = [ +PACKAGES = maybe_argparse + [ + # Pip has no dependencies, as it vendors everything: ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' 'pip-{0}.tar.gz'.format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] - - -OTHER_PACKAGES = maybe_argparse + [ + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' 'setuptools-29.0.1.tar.gz', @@ -1379,21 +1376,16 @@ def main(): index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - # We download and install pip first, then the rest, to avoid the bug - # https://github.com/certbot/certbot/issues/4938. - pip_downloads, other_downloads = [ - [hashed_download(index_base + '/packages/' + path, - temp, - digest) - for path, digest in packages] - for packages in (PIP_PACKAGE, OTHER_PACKAGES)] - for downloads in (pip_downloads, other_downloads): - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it - # otherwise sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + downloads = [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in PACKAGES] + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it otherwise + # sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index ed55b37e9..d55d5bceb 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 +"""A small script that can act as a trust root for installing pip >=8 Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, @@ -57,7 +57,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -70,14 +70,11 @@ maybe_argparse = ( if version_info < (2, 7, 0) else []) -# Pip has no dependencies, as it vendors everything: -PIP_PACKAGE = [ +PACKAGES = maybe_argparse + [ + # Pip has no dependencies, as it vendors everything: ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' 'pip-{0}.tar.gz'.format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] - - -OTHER_PACKAGES = maybe_argparse + [ + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' 'setuptools-29.0.1.tar.gz', @@ -162,21 +159,16 @@ def main(): index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - # We download and install pip first, then the rest, to avoid the bug - # https://github.com/certbot/certbot/issues/4938. - pip_downloads, other_downloads = [ - [hashed_download(index_base + '/packages/' + path, - temp, - digest) - for path, digest in packages] - for packages in (PIP_PACKAGE, OTHER_PACKAGES)] - for downloads in (pip_downloads, other_downloads): - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it - # otherwise sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + downloads = [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in PACKAGES] + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it otherwise + # sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: From f4bac423fb794c14e426defecc492306ea53cbc4 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Wed, 7 Mar 2018 15:09:47 -0800 Subject: [PATCH 53/56] fix(acme): client._revoke sends default content_type (#5687) --- acme/acme/client.py | 3 +-- acme/acme/client_test.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index d52c82a5c..9e2478afe 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -227,8 +227,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response = self._post(url, messages.Revocation( certificate=cert, - reason=rsn), - content_type=None) + reason=rsn)) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index a0c27e74f..00b9e19dd 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -635,8 +635,7 @@ class ClientTest(ClientTestBase): def test_revoke(self): self.client.revoke(self.certr.body, self.rsn) self.net.post.assert_called_once_with( - self.directory[messages.Revocation], mock.ANY, content_type=None, - acme_version=1) + self.directory[messages.Revocation], mock.ANY, acme_version=1) def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) @@ -776,8 +775,7 @@ class ClientV2Test(ClientTestBase): def test_revoke(self): self.client.revoke(messages_test.CERT, self.rsn) self.net.post.assert_called_once_with( - self.directory["revokeCert"], mock.ANY, content_type=None, - acme_version=2) + self.directory["revokeCert"], mock.ANY, acme_version=2) class MockJSONDeSerializable(jose.JSONDeSerializable): From cc18da926ed0c64ffd9564bcbf8cc701f6506360 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Mar 2018 11:09:31 -0800 Subject: [PATCH 54/56] Quiet pylint (#5689) --- certbot-dns-google/certbot_dns_google/dns_google_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index afab847cf..72b8be8af 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -278,6 +278,7 @@ class GoogleClientTest(unittest.TestCase): def test_get_existing_fallback(self, unused_credential_mock): client, unused_changes = self._setUp_client_with_mock( [{'managedZones': [{'id': self.zone}]}]) + # pylint: disable=no-member mock_execute = client.dns.resourceRecordSets.return_value.list.return_value.execute mock_execute.side_effect = API_ERROR From cc24b4e40af5841c8dfddeecd9bde7d1acce62e8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Mar 2018 11:12:33 -0800 Subject: [PATCH 55/56] Fix --allow-subset-of-names (#5690) * Remove aauthzr instance variable * If domain begins with fail, fail the challenge. * test --allow-subset-of-names * Fix renewal and add extra check * test after hook checks --- certbot/auth_handler.py | 80 ++++++++++++++++-------------- certbot/tests/auth_handler_test.py | 54 ++++++++++---------- tests/boulder-integration.sh | 13 +++++ tests/manual-dns-auth.sh | 12 +++-- tests/manual-dns-cleanup.sh | 11 ++-- 5 files changed, 99 insertions(+), 71 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 67d36c8cc..51cdf09ee 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -34,8 +34,6 @@ class AuthHandler(object): :ivar account: Client's Account :type account: :class:`certbot.account.Account` - :ivar aauthzrs: ACME Authorization Resources and their active challenges - :type aauthzrs: `list` of `AnnotatedAuthzr` :ivar list pref_challs: sorted user specified preferred challenges type strings with the most preferred challenge listed first @@ -45,7 +43,6 @@ class AuthHandler(object): self.acme = acme self.account = account - self.aauthzrs = [] self.pref_challs = pref_challs def handle_authorizations(self, orderr, best_effort=False): @@ -63,29 +60,29 @@ class AuthHandler(object): authorizations """ - for authzr in orderr.authorizations: - self.aauthzrs.append(AnnotatedAuthzr(authzr, [])) + aauthzrs = [AnnotatedAuthzr(authzr, []) + for authzr in orderr.authorizations] - self._choose_challenges() + self._choose_challenges(aauthzrs) config = zope.component.getUtility(interfaces.IConfig) notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... - while self._has_challenges(): - resp = self._solve_challenges() + while self._has_challenges(aauthzrs): + resp = self._solve_challenges(aauthzrs) logger.info("Waiting for verification...") if config.debug_challenges: notify('Challenges loaded. Press continue to submit to CA. ' 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp, best_effort) + self._respond(aauthzrs, resp, best_effort) # Just make sure all decisions are complete. - self.verify_authzr_complete() + self.verify_authzr_complete(aauthzrs) # Only return valid authorizations - retVal = [aauthzr.authzr for aauthzr in self.aauthzrs + retVal = [aauthzr.authzr for aauthzr in aauthzrs if aauthzr.authzr.body.status == messages.STATUS_VALID] if not retVal: @@ -94,10 +91,10 @@ class AuthHandler(object): return retVal - def _choose_challenges(self): + def _choose_challenges(self, aauthzrs): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: aauthzr_challenges = aauthzr.authzr.body.challenges if self.acme.acme_version == 1: combinations = aauthzr.authzr.body.combinations @@ -113,15 +110,15 @@ class AuthHandler(object): aauthzr.authzr, path) aauthzr.achalls.extend(aauthzr_achalls) - def _has_challenges(self): + def _has_challenges(self, aauthzrs): """Do we have any challenges to perform?""" - return any(aauthzr.achalls for aauthzr in self.aauthzrs) + return any(aauthzr.achalls for aauthzr in aauthzrs) - def _solve_challenges(self): + def _solve_challenges(self, aauthzrs): """Get Responses for challenges from authenticators.""" resp = [] - all_achalls = self._get_all_achalls() - with error_handler.ErrorHandler(self._cleanup_challenges): + all_achalls = self._get_all_achalls(aauthzrs) + with error_handler.ErrorHandler(self._cleanup_challenges, all_achalls): try: if all_achalls: resp = self.auth.perform(all_achalls) @@ -134,15 +131,15 @@ class AuthHandler(object): return resp - def _get_all_achalls(self): + def _get_all_achalls(self, aauthzrs): """Return all active challenges.""" all_achalls = [] - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: all_achalls.extend(aauthzr.achalls) return all_achalls - def _respond(self, resp, best_effort): + def _respond(self, aauthzrs, resp, best_effort): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -150,24 +147,27 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 chall_update = dict() - active_achalls = self._send_responses(resp, chall_update) + active_achalls = self._send_responses(aauthzrs, resp, chall_update) # Check for updated status... try: - self._poll_challenges(chall_update, best_effort) + self._poll_challenges(aauthzrs, chall_update, best_effort) finally: - self._cleanup_challenges(active_achalls) + self._cleanup_challenges(aauthzrs, active_achalls) - def _send_responses(self, resps, chall_update): + def _send_responses(self, aauthzrs, resps, chall_update): """Send responses and make sure errors are handled. + :param aauthzrs: authorizations and the selected annotated challenges + to try and perform + :type aauthzrs: `list` of `AnnotatedAuthzr` :param dict chall_update: parameter that is updated to hold aauthzr index to list of outstanding solved annotated challenges """ active_achalls = [] resps_iter = iter(resps) - for i, aauthzr in enumerate(self.aauthzrs): + for i, aauthzr in enumerate(aauthzrs): for achall in aauthzr.achalls: # This line needs to be outside of the if block below to # ensure failed challenges are cleaned up correctly @@ -184,8 +184,8 @@ class AuthHandler(object): return active_achalls - def _poll_challenges( - self, chall_update, best_effort, min_sleep=3, max_rounds=15): + def _poll_challenges(self, aauthzrs, chall_update, + best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" indices_to_check = set(chall_update.keys()) comp_indices = set() @@ -197,7 +197,7 @@ class AuthHandler(object): all_failed_achalls = set() for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( - index, chall_update[index]) + aauthzrs, index, chall_update[index]) if len(comp_achalls) == len(chall_update[index]): comp_indices.add(index) @@ -210,7 +210,7 @@ class AuthHandler(object): comp_indices.add(index) logger.warning( "Challenge failed for domain %s", - self.aauthzrs[index].authzr.body.identifier.value) + aauthzrs[index].authzr.body.identifier.value) else: all_failed_achalls.update( updated for _, updated in failed_achalls) @@ -223,14 +223,14 @@ class AuthHandler(object): comp_indices.clear() rounds += 1 - def _handle_check(self, index, achalls): + def _handle_check(self, aauthzrs, index, achalls): """Returns tuple of ('completed', 'failed').""" completed = [] failed = [] - original_aauthzr = self.aauthzrs[index] + original_aauthzr = aauthzrs[index] updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) - self.aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) + aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) if updated_authzr.body.status == messages.STATUS_VALID: return achalls, [] @@ -287,7 +287,7 @@ class AuthHandler(object): chall_prefs.extend(plugin_pref) return chall_prefs - def _cleanup_challenges(self, achall_list=None): + def _cleanup_challenges(self, aauthzrs, achall_list=None): """Cleanup challenges. If achall_list is not provided, cleanup all achallenges. @@ -296,26 +296,30 @@ class AuthHandler(object): logger.info("Cleaning up challenges") if achall_list is None: - achalls = self._get_all_achalls() + achalls = self._get_all_achalls(aauthzrs) else: achalls = achall_list if achalls: self.auth.cleanup(achalls) for achall in achalls: - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: if achall in aauthzr.achalls: aauthzr.achalls.remove(achall) break - def verify_authzr_complete(self): + def verify_authzr_complete(self, aauthzrs): """Verifies that all authorizations have been decided. + :param aauthzrs: authorizations and their selected annotated + challenges + :type aauthzrs: `list` of `AnnotatedAuthzr` + :returns: Whether all authzr are complete :rtype: bool """ - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: authzr = aauthzr.authzr if (authzr.body.status != messages.STATUS_VALID and authzr.body.status != messages.STATUS_INVALID): diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index b6af3d0f5..54e284d9e 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -101,7 +101,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) @@ -132,7 +132,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_net.answer_challenge.call_count, 3) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) @@ -158,7 +158,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) @@ -187,7 +187,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # Check poll call self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(len(list(six.iterkeys(chall_update))), 3) self.assertTrue(0 in list(six.iterkeys(chall_update))) self.assertEqual(len(chall_update[0]), 1) @@ -278,8 +278,8 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1, unused_2): - for i, aauthzr in enumerate(self.handler.aauthzrs): + def _validate_all(self, aauthzrs, unused_1, unused_2): + for i, aauthzr in enumerate(aauthzrs): azr = aauthzr.authzr updated_azr = acme_util.gen_authzr( messages.STATUS_VALID, @@ -287,7 +287,7 @@ class HandleAuthorizationsTest(unittest.TestCase): [challb.chall for challb in azr.body.challenges], [messages.STATUS_VALID] * len(azr.body.challenges), azr.body.combinations) - self.handler.aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) + aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) class PollChallengesTest(unittest.TestCase): @@ -304,19 +304,21 @@ class PollChallengesTest(unittest.TestCase): None, self.mock_net, mock.Mock(key="mock_key"), []) self.doms = ["0", "1", "2"] - self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[0], - [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False), [])) - self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) - self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) + self.aauthzrs = [ + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[0], + [acme_util.HTTP01, acme_util.TLSSNI01], + [messages.STATUS_PENDING] * 2, False), []), + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[1], + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []), + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[2], + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []) + ] self.chall_update = {} - for i, aauthzr in enumerate(self.handler.aauthzrs): + for i, aauthzr in enumerate(self.aauthzrs): self.chall_update[i] = [ challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) for challb in aauthzr.authzr.body.challenges] @@ -324,17 +326,17 @@ class PollChallengesTest(unittest.TestCase): @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.handler._poll_challenges(self.chall_update, False) + self.handler._poll_challenges(self.aauthzrs, self.chall_update, False) - for aauthzr in self.handler.aauthzrs: + for aauthzr in self.aauthzrs: self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) @mock.patch("certbot.auth_handler.time") def test_poll_challenges_failure_best_effort(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid - self.handler._poll_challenges(self.chall_update, True) + self.handler._poll_challenges(self.aauthzrs, self.chall_update, True) - for aauthzr in self.handler.aauthzrs: + for aauthzr in self.aauthzrs: self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) @mock.patch("certbot.auth_handler.time") @@ -343,7 +345,7 @@ class PollChallengesTest(unittest.TestCase): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, - self.chall_update, False) + self.aauthzrs, self.chall_update, False) @mock.patch("certbot.auth_handler.time") def test_unable_to_find_challenge_status(self, unused_mock_time): @@ -353,11 +355,11 @@ class PollChallengesTest(unittest.TestCase): challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, - self.chall_update, False) + self.aauthzrs, self.chall_update, False) def test_verify_authzr_failure(self): - self.assertRaises( - errors.AuthorizationError, self.handler.verify_authzr_complete) + self.assertRaises(errors.AuthorizationError, + self.handler.verify_authzr_complete, self.aauthzrs) def _mock_poll_solve_one_valid(self, authzr): # Pending here because my dummy script won't change the full status. diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 2b92476fd..9748befa3 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -327,6 +327,19 @@ CheckDirHooks 1 common renew --cert-name le2.wtf CheckDirHooks 1 +# manual-dns-auth.sh will skip completing the challenge for domains that begin +# with fail. +common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ + --allow-subset-of-names \ + --preferred-challenges dns,tls-sni \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh + +if common certificates | grep "fail\.dns1\.le\.wtf"; then + echo "certificate should not have been issued for domain!" >&2 + exit 1 +fi + # ECDSA openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ diff --git a/tests/manual-dns-auth.sh b/tests/manual-dns-auth.sh index 9b9a1a5eb..febecf455 100755 --- a/tests/manual-dns-auth.sh +++ b/tests/manual-dns-auth.sh @@ -1,4 +1,8 @@ -#!/bin/sh -curl -X POST 'http://localhost:8055/set-txt' -d \ - "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\", \ - \"value\": \"$CERTBOT_VALIDATION\"}" +#!/bin/bash + +# If domain begins with fail, fail the challenge by not completing it. +if [[ "$CERTBOT_DOMAIN" != fail* ]]; then + curl -X POST 'http://localhost:8055/set-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\", \ + \"value\": \"$CERTBOT_VALIDATION\"}" +fi diff --git a/tests/manual-dns-cleanup.sh b/tests/manual-dns-cleanup.sh index 0c5c56b17..1c09e892c 100755 --- a/tests/manual-dns-cleanup.sh +++ b/tests/manual-dns-cleanup.sh @@ -1,3 +1,8 @@ -#!/bin/sh -curl -X POST 'http://localhost:8055/clear-txt' -d \ - "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}" +#!/bin/bash + +# If domain begins with fail, we didn't complete the challenge so there is +# nothing to clean up. +if [[ "$CERTBOT_DOMAIN" != fail* ]]; then + curl -X POST 'http://localhost:8055/clear-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}" +fi From 2e6d65d9ecb5f2416413597d74c3599b470a5bd4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Mar 2018 17:24:30 -0800 Subject: [PATCH 56/56] Add readthedocs requirements files (#5696) * Add readthedocs requirements files. * Only install docs extras for plugin. --- .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-google/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-luadns/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-nsone/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-rfc2136/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-route53/readthedocs.org.requirements.txt | 12 ++++++++++++ 10 files changed, 120 insertions(+) create mode 100644 certbot-dns-cloudflare/readthedocs.org.requirements.txt create mode 100644 certbot-dns-cloudxns/readthedocs.org.requirements.txt create mode 100644 certbot-dns-digitalocean/readthedocs.org.requirements.txt create mode 100644 certbot-dns-dnsimple/readthedocs.org.requirements.txt create mode 100644 certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt create mode 100644 certbot-dns-google/readthedocs.org.requirements.txt create mode 100644 certbot-dns-luadns/readthedocs.org.requirements.txt create mode 100644 certbot-dns-nsone/readthedocs.org.requirements.txt create mode 100644 certbot-dns-rfc2136/readthedocs.org.requirements.txt create mode 100644 certbot-dns-route53/readthedocs.org.requirements.txt diff --git a/certbot-dns-cloudflare/readthedocs.org.requirements.txt b/certbot-dns-cloudflare/readthedocs.org.requirements.txt new file mode 100644 index 000000000..b18901111 --- /dev/null +++ b/certbot-dns-cloudflare/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-cloudflare[docs] diff --git a/certbot-dns-cloudxns/readthedocs.org.requirements.txt b/certbot-dns-cloudxns/readthedocs.org.requirements.txt new file mode 100644 index 000000000..ae2ff8165 --- /dev/null +++ b/certbot-dns-cloudxns/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-cloudxns[docs] diff --git a/certbot-dns-digitalocean/readthedocs.org.requirements.txt b/certbot-dns-digitalocean/readthedocs.org.requirements.txt new file mode 100644 index 000000000..08d973ab3 --- /dev/null +++ b/certbot-dns-digitalocean/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-digitalocean[docs] diff --git a/certbot-dns-dnsimple/readthedocs.org.requirements.txt b/certbot-dns-dnsimple/readthedocs.org.requirements.txt new file mode 100644 index 000000000..fef73916c --- /dev/null +++ b/certbot-dns-dnsimple/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-dnsimple[docs] diff --git a/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt b/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt new file mode 100644 index 000000000..8f8c6c731 --- /dev/null +++ b/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-dnsmadeeasy[docs] diff --git a/certbot-dns-google/readthedocs.org.requirements.txt b/certbot-dns-google/readthedocs.org.requirements.txt new file mode 100644 index 000000000..6ea393f86 --- /dev/null +++ b/certbot-dns-google/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-google[docs] diff --git a/certbot-dns-luadns/readthedocs.org.requirements.txt b/certbot-dns-luadns/readthedocs.org.requirements.txt new file mode 100644 index 000000000..acb51e4ef --- /dev/null +++ b/certbot-dns-luadns/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-luadns[docs] diff --git a/certbot-dns-nsone/readthedocs.org.requirements.txt b/certbot-dns-nsone/readthedocs.org.requirements.txt new file mode 100644 index 000000000..dbdee4480 --- /dev/null +++ b/certbot-dns-nsone/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-nsone[docs] diff --git a/certbot-dns-rfc2136/readthedocs.org.requirements.txt b/certbot-dns-rfc2136/readthedocs.org.requirements.txt new file mode 100644 index 000000000..df89018ce --- /dev/null +++ b/certbot-dns-rfc2136/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-rfc2136[docs] diff --git a/certbot-dns-route53/readthedocs.org.requirements.txt b/certbot-dns-route53/readthedocs.org.requirements.txt new file mode 100644 index 000000000..660a90d0e --- /dev/null +++ b/certbot-dns-route53/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-route53[docs]