From cff8769db737f19a93fe26cd1d5247904c4b188a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 13 Feb 2019 18:37:01 +0200 Subject: [PATCH 01/44] Apache: respect CERTBOT_DOCS environment variable (#6598) Apache plugin will now use command line default values from `ApacheConfingurator.OS_DEFAULTS` instead of respective distribution override when `CERTBOT_DOCS=1` environment variable is present. Fixes: #6234 * Apache: respect CERTBOT_DOCS environment variable * Move the tests to apache plugin --- CHANGELOG.md | 3 ++ certbot-apache/certbot_apache/configurator.py | 31 +++++++++++++------ .../certbot_apache/tests/configurator_test.py | 31 +++++++++++++++++++ 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac0f60eb8..57a895e07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Certbot and its acme module now depend on josepy>=1.1.0 to avoid printing the warnings described at https://github.com/certbot/josepy/issues/13. +* Apache plugin now respects CERTBOT_DOCS environment variable when adding + command line defaults. ### Fixed @@ -23,6 +25,7 @@ package with changes other than its version number was: * acme * certbot +* certbot-apache More details about these changes can be found on our GitHub repo. diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index efd766e63..f7f4ff925 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -92,6 +92,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ description = "Apache Web Server plugin" + if os.environ.get("CERTBOT_DOCS") == "1": + description += ( # pragma: no cover + " (Please note that the default values of the Apache plugin options" + " change depending on the operating system Certbot is run on.)" + ) OS_DEFAULTS = dict( server_root="/etc/apache2", @@ -141,28 +146,36 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # When adding, modifying or deleting command line arguments, be sure to # include the changes in the list used in method _prepare_options() to # ensure consistent behavior. - add("enmod", default=cls.OS_DEFAULTS["enmod"], + + # Respect CERTBOT_DOCS environment variable and use default values from + # base class regardless of the underlying distribution (overrides). + if os.environ.get("CERTBOT_DOCS") == "1": + DEFAULTS = ApacheConfigurator.OS_DEFAULTS + else: + # cls.OS_DEFAULTS can be distribution specific, see override classes + DEFAULTS = cls.OS_DEFAULTS + add("enmod", default=DEFAULTS["enmod"], help="Path to the Apache 'a2enmod' binary") - add("dismod", default=cls.OS_DEFAULTS["dismod"], + add("dismod", default=DEFAULTS["dismod"], help="Path to the Apache 'a2dismod' binary") - add("le-vhost-ext", default=cls.OS_DEFAULTS["le_vhost_ext"], + add("le-vhost-ext", default=DEFAULTS["le_vhost_ext"], help="SSL vhost configuration extension") - add("server-root", default=cls.OS_DEFAULTS["server_root"], + add("server-root", default=DEFAULTS["server_root"], help="Apache server root directory") add("vhost-root", default=None, help="Apache server VirtualHost configuration root") - add("logs-root", default=cls.OS_DEFAULTS["logs_root"], + add("logs-root", default=DEFAULTS["logs_root"], help="Apache server logs directory") add("challenge-location", - default=cls.OS_DEFAULTS["challenge_location"], + default=DEFAULTS["challenge_location"], help="Directory path for challenge configuration") - add("handle-modules", default=cls.OS_DEFAULTS["handle_modules"], + add("handle-modules", default=DEFAULTS["handle_modules"], help="Let installer handle enabling required modules for you " + "(Only Ubuntu/Debian currently)") - add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"], + add("handle-sites", default=DEFAULTS["handle_sites"], help="Let installer handle enabling sites for you " + "(Only Ubuntu/Debian currently)") - add("ctl", default=cls.OS_DEFAULTS["ctl"], + add("ctl", default=DEFAULTS["ctl"], help="Full path to Apache control script") util.add_deprecated_argument( add, argument_name="init-script", nargs=1) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index ff93682b6..7c281d707 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -115,6 +115,37 @@ class MultipleVhostsTest(util.ApacheTest): # Weak test.. ApacheConfigurator.add_parser_arguments(mock.MagicMock()) + def test_docs_parser_arguments(self): + os.environ["CERTBOT_DOCS"] = "1" + from certbot_apache.configurator import ApacheConfigurator + mock_add = mock.MagicMock() + ApacheConfigurator.add_parser_arguments(mock_add) + parserargs = ["server_root", "enmod", "dismod", "le_vhost_ext", + "vhost_root", "logs_root", "challenge_location", + "handle_modules", "handle_sites", "ctl"] + exp = dict() + + for k in ApacheConfigurator.OS_DEFAULTS: + if k in parserargs: + exp[k.replace("_", "-")] = ApacheConfigurator.OS_DEFAULTS[k] + # Special cases + exp["vhost-root"] = None + exp["init-script"] = None + + found = set() + for call in mock_add.call_args_list: + # init-script is a special case: deprecated argument + if call[0][0] != "init-script": + self.assertEqual(exp[call[0][0]], call[1]['default']) + found.add(call[0][0]) + + # Make sure that all (and only) the expected values exist + self.assertEqual(len(mock_add.call_args_list), len(found)) + for e in exp: + self.assertTrue(e in found) + + del os.environ["CERTBOT_DOCS"] + def test_add_parser_arguments_all_configurators(self): # pylint: disable=no-self-use from certbot_apache.entrypoint import OVERRIDE_CLASSES for cls in OVERRIDE_CLASSES.values(): From acc0b1e773de076e71f628b0b64e69f85fc9a39b Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 14 Feb 2019 19:43:27 +0100 Subject: [PATCH 02/44] Fix the pebble fetch script (#6765) This PR updates and fixes `pebble-fetch.sh` considering latest improvements done on Pebble, to start a working instance. * Fix the pebble fetch script * Update pebble-fetch.sh * Update tox.ini --- tests/pebble-fetch.sh | 33 +++++++++++++++++++++++++-------- tox.ini | 1 - 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/tests/pebble-fetch.sh b/tests/pebble-fetch.sh index b0ba08961..6b562eec2 100755 --- a/tests/pebble-fetch.sh +++ b/tests/pebble-fetch.sh @@ -2,7 +2,7 @@ # Download and run Pebble instance for integration testing set -xe -PEBBLE_VERSION=2018-11-02 +PEBBLE_VERSION=v1.0.1 # We reuse the same GOPATH-style directory than for Boulder. # Pebble does not need it, but it will make the installation consistent with Boulder's one. @@ -13,15 +13,32 @@ mkdir -p ${PEBBLEPATH} cat << UNLIKELY_EOF > "$PEBBLEPATH/docker-compose.yml" version: '3' - services: - pebble: - image: letsencrypt/pebble:${PEBBLE_VERSION} - command: pebble -strict ${PEBBLE_STRICT:-false} -dnsserver 10.77.77.1 - ports: - - 14000:14000 - environment: + pebble: + image: letsencrypt/pebble:${PEBBLE_VERSION} + command: pebble -dnsserver 10.30.50.3:8053 + environment: - PEBBLE_VA_NOSLEEP=1 + ports: + - 14000:14000 + networks: + acmenet: + ipv4_address: 10.30.50.2 + challtestsrv: + image: letsencrypt/pebble-challtestsrv:${PEBBLE_VERSION} + command: pebble-challtestsrv -defaultIPv6 "" -defaultIPv4 10.30.50.1 + ports: + - 8055:8055 + networks: + acmenet: + ipv4_address: 10.30.50.3 +networks: + acmenet: + driver: bridge + ipam: + driver: default + config: + - subnet: 10.30.50.0/24 UNLIKELY_EOF docker-compose -f "$PEBBLEPATH/docker-compose.yml" up -d pebble diff --git a/tox.ini b/tox.ini index b66e330da..b386ebc86 100644 --- a/tox.ini +++ b/tox.ini @@ -166,7 +166,6 @@ passenv = HOME GOPATH PEBBLEPATH - PEBBLE_STRICT setenv = SERVER=https://localhost:14000/dir From 583d40f5cf5bc22799eadb99b9900e4cff17b4a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 14 Feb 2019 15:26:44 -0800 Subject: [PATCH 03/44] Pin pytest in test_sdists.sh. (#6764) * pin pytest in test_sdists.sh. * Use pip_install.py in test_tests.sh. --- tests/letstest/scripts/test_sdists.sh | 2 ++ tests/letstest/scripts/test_tests.sh | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index 0b9a91ffd..260a0acfb 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -12,6 +12,8 @@ export VENV_ARGS="-p $PYTHON" # setup venv tools/_venv_common.py --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt . ./venv/bin/activate +# pytest is needed to run tests on some of our packages so we install a pinned version here. +tools/pip_install.py pytest # build sdists for pkg_dir in acme . $PLUGINS; do diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index e6ab836b8..d5fd6e14a 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -1,18 +1,20 @@ #!/bin/sh -xe -LE_AUTO="letsencrypt/letsencrypt-auto-source/letsencrypt-auto" +REPO_ROOT="letsencrypt" +LE_AUTO="$REPO_ROOT/letsencrypt-auto-source/letsencrypt-auto" LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" MODULES="acme certbot certbot_apache certbot_nginx" +PIP_INSTALL="$REPO_ROOT/tools/pip_install.py" VENV_NAME=venv # *-auto respects VENV_PATH $LE_AUTO --os-packages-only LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version . $VENV_NAME/bin/activate +"$PIP_INSTALL" pytest # change to an empty directory to ensure CWD doesn't affect tests cd $(mktemp -d) -pip install pytest==3.2.5 for module in $MODULES ; do echo testing $module From e40d929e804137566c0e04f87f9d5c62a7966a4a Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 15 Feb 2019 01:55:27 +0100 Subject: [PATCH 04/44] [Windows|Unix] New platform independent locking mechanism - the revenge (#6663) First PR about this issue, #6440, involved to much refactoring to ensure a correct behavior on Linux and installer plugins. This PR proposes a new implementation of a lock mechanism for Linux and Windows, without implying a refactoring. It takes strictly the existing behavior for Linux, and add the appropriate logic for Windows. The `lock` module formalizes two independant mechanism dedicated to each platform, to improve maintainability. Tests related to locking are re-activated for Windows, or definitively skipped because of irrelevancy for this platform. 6 more tests are enabled overall. * Reimplement lock file on basic level * Remove unused code * Re-activate some tests * Update doc * Reactivate tests relevant to locks in Windows. Correct a test that was not testing what is supposed to test. * Clean compat. * Move close sooner in Windows lock implementation * Add strong mypy types * Use os.name * Refactor lock mechanism logic * Enable more tests * Update lock.py * Update lock_test.py --- certbot/compat.py | 56 ---------- certbot/lock.py | 213 +++++++++++++++++++++++++++++-------- certbot/tests/lock_test.py | 57 ++++++---- certbot/tests/util_test.py | 16 +-- 4 files changed, 216 insertions(+), 126 deletions(-) diff --git a/certbot/compat.py b/certbot/compat.py index 3b5d068a6..f533f5954 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -13,13 +13,6 @@ import stat from certbot import errors -try: - # Linux specific - import fcntl # pylint: disable=import-error -except ImportError: - # Windows specific - import msvcrt # pylint: disable=import-error - UNPRIVILEGED_SUBCOMMANDS_ALLOWED = [ 'certificates', 'enhance', 'revoke', 'delete', 'register', 'unregister', 'config_changes', 'plugins'] @@ -118,55 +111,6 @@ def readline_with_timeout(timeout, prompt): return sys.stdin.readline() -def lock_file(fd): - """ - Lock the file linked to the specified file descriptor. - - :param int fd: The file descriptor of the file to lock. - - """ - if 'fcntl' in sys.modules: - # Linux specific - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - else: - # Windows specific - msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) - - -def release_locked_file(fd, path): - """ - Remove, close, and release a lock file specified by its file descriptor and its path. - - :param int fd: The file descriptor of the lock file. - :param str path: The path of the lock file. - - """ - # Linux specific - # - # It is important the lock file is removed before it's released, - # otherwise: - # - # process A: open lock file - # process B: release lock file - # process A: lock file - # process A: check device and inode - # process B: delete file - # process C: open and lock a different file at the same path - try: - os.remove(path) - except OSError as err: - if err.errno == errno.EACCES: - # Windows specific - # We will not be able to remove a file before closing it. - # To avoid race conditions described for Linux, we will not delete the lockfile, - # just close it to be reused on the next Certbot call. - pass - else: - raise - finally: - os.close(fd) - - def compare_file_modes(mode1, mode2): """Return true if the two modes can be considered as equals for this platform""" if os.name != 'nt': diff --git a/certbot/lock.py b/certbot/lock.py index 3ff46518d..760a12b8f 100644 --- a/certbot/lock.py +++ b/certbot/lock.py @@ -1,15 +1,23 @@ -"""Implements file locks for locking files and directories in UNIX.""" +"""Implements file locks compatible with Linux and Windows for locking files and directories.""" import errno import logging import os +try: + import fcntl # pylint: disable=import-error +except ImportError: + import msvcrt # pylint: disable=import-error + POSIX_MODE = False +else: + POSIX_MODE = True -from certbot import compat from certbot import errors +from acme.magic_typing import Optional, Callable # pylint: disable=unused-import, no-name-in-module logger = logging.getLogger(__name__) def lock_dir(dir_path): + # type: (str) -> LockFile """Place a lock file on the directory at dir_path. The lock file is placed in the root of dir_path with the name @@ -27,34 +35,99 @@ def lock_dir(dir_path): class LockFile(object): - """A UNIX lock file. - - This lock file is released when the locked file is closed or the - process exits. It cannot be used to provide synchronization between - threads. It is based on the lock_file package by Martin Horcicka. - + """ + Platform independent file lock system. + LockFile accepts a parameter, the path to a file acting as a lock. Once the LockFile, + instance is created, the associated file is 'locked from the point of view of the OS, + meaning that if another instance of Certbot try at the same time to acquire the same lock, + it will raise an Exception. Calling release method will release the lock, and make it + available to every other instance. + Upon exit, Certbot will also release all the locks. + This allows us to protect a file or directory from being concurrently accessed + or modified by two Certbot instances. + LockFile is platform independent: it will proceed to the appropriate OS lock mechanism + depending on Linux or Windows. """ def __init__(self, path): - """Initialize and acquire the lock file. - - :param str path: path to the file to lock - - :raises errors.LockError: if unable to acquire the lock - + # type: (str) -> None + """ + Create a LockFile instance on the given file path, and acquire lock. + :param str path: the path to the file that will hold a lock """ - super(LockFile, self).__init__() self._path = path - self._fd = None + mechanism = _UnixLockMechanism if POSIX_MODE else _WindowsLockMechanism + self._lock_mechanism = mechanism(path) self.acquire() + def __repr__(self): + # type: () -> str + repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) + if self.is_locked(): + repr_str += 'acquired>' + else: + repr_str += 'released>' + return repr_str + def acquire(self): - """Acquire the lock file. - - :raises errors.LockError: if lock is already held - :raises OSError: if unable to open or stat the lock file - + # type: () -> None """ + Acquire the lock on the file, forbidding any other Certbot instance to acquire it. + :raises errors.LockError: if unable to acquire the lock + """ + self._lock_mechanism.acquire() + + def release(self): + # type: () -> None + """ + Release the lock on the file, allowing any other Certbot instance to acquire it. + """ + self._lock_mechanism.release() + + def is_locked(self): + # type: () -> bool + """ + Check if the file is currently locked. + :return: True if the file is locked, False otherwise + """ + return self._lock_mechanism.is_locked() + + +class _BaseLockMechanism(object): + def __init__(self, path): + # type: (str) -> None + """ + Create a lock file mechanism for Unix. + :param str path: the path to the lock file + """ + self._path = path + self._fd = None # type: Optional[int] + + def is_locked(self): + # type: () -> bool + """Check if lock file is currently locked. + :return: True if the lock file is locked + :rtype: bool + """ + return self._fd is not None + + def acquire(self): # pylint: disable=missing-docstring + pass # pragma: no cover + + def release(self): # pylint: disable=missing-docstring + pass # pragma: no cover + + +class _UnixLockMechanism(_BaseLockMechanism): + """ + A UNIX lock file mechanism. + This lock file is released when the locked file is closed or the + process exits. It cannot be used to provide synchronization between + threads. It is based on the lock_file package by Martin Horcicka. + """ + def acquire(self): + # type: () -> None + """Acquire the lock.""" while self._fd is None: # Open the file fd = os.open(self._path, os.O_CREAT | os.O_WRONLY, 0o600) @@ -68,33 +141,29 @@ class LockFile(object): os.close(fd) def _try_lock(self, fd): - """Try to acquire the lock file without blocking. - + # type: (int) -> None + """ + Try to acquire the lock file without blocking. :param int fd: file descriptor of the opened file to lock - """ try: - compat.lock_file(fd) + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as err: if err.errno in (errno.EACCES, errno.EAGAIN): - logger.debug( - "A lock on %s is held by another process.", self._path) - raise errors.LockError( - "Another instance of Certbot is already running.") + logger.debug('A lock on %s is held by another process.', self._path) + raise errors.LockError('Another instance of Certbot is already running.') raise def _lock_success(self, fd): - """Did we successfully grab the lock? - + # type: (int) -> bool + """ + Did we successfully grab the lock? Because this class deletes the locked file when the lock is released, it is possible another process removed and recreated the file between us opening the file and acquiring the lock. - :param int fd: file descriptor of the opened file to lock - :returns: True if the lock was successfully acquired :rtype: bool - """ try: stat1 = os.stat(self._path) @@ -108,17 +177,75 @@ class LockFile(object): # the same device and inode, they're the same file. return stat1.st_dev == stat2.st_dev and stat1.st_ino == stat2.st_ino - def __repr__(self): - repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) - if self._fd is None: - repr_str += 'released>' - else: - repr_str += 'acquired>' - return repr_str + def release(self): + # type: () -> None + """Remove, close, and release the lock file.""" + # It is important the lock file is removed before it's released, + # otherwise: + # + # process A: open lock file + # process B: release lock file + # process A: lock file + # process A: check device and inode + # process B: delete file + # process C: open and lock a different file at the same path + try: + os.remove(self._path) + finally: + # Following check is done to make mypy happy: it ensure that self._fd, marked + # as Optional[int] is effectively int to make it compatible with os.close signature. + if self._fd is None: # pragma: no cover + raise TypeError('Error, self._fd is None.') + try: + os.close(self._fd) + finally: + self._fd = None + + +class _WindowsLockMechanism(_BaseLockMechanism): + """ + A Windows lock file mechanism. + By default on Windows, acquiring a file handler gives exclusive access to the process + and results in an effective lock. However, it is possible to explicitly acquire the + file handler in shared access in terms of read and write, and this is done by os.open + and io.open in Python. So an explicit lock needs to be done through the call of + msvcrt.locking, that will lock the first byte of the file. In theory, it is also + possible to access a file in shared delete access, allowing other processes to delete an + opened file. But this needs also to be done explicitly by all processes using the Windows + low level APIs, and Python does not do it. As of Python 3.7 and below, Python developers + state that deleting a file opened by a process from another process is not possible with + os.open and io.open. + Consequently, mscvrt.locking is sufficient to obtain an effective lock, and the race + condition encountered on Linux is not possible on Windows, leading to a simpler workflow. + """ + def acquire(self): + """Acquire the lock""" + open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC + + fd = os.open(self._path, open_mode, 0o600) + try: + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + except (IOError, OSError) as err: + os.close(fd) + # Anything except EACCES is unexpected. Raise directly the error in that case. + if err.errno != errno.EACCES: + raise + logger.debug('A lock on %s is held by another process.', self._path) + raise errors.LockError('Another instance of Certbot is already running.') + + self._fd = fd def release(self): - """Remove, close, and release the lock file.""" + """Release the lock.""" try: - compat.release_locked_file(self._fd, self._path) + msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) + os.close(self._fd) + + try: + os.remove(self._path) + except OSError as e: + # If the lock file cannot be removed, it is not a big deal. + # Likely another instance is acquiring the lock we just released. + logger.debug(str(e)) finally: self._fd = None diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index aa82701f3..aa1de299b 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -3,6 +3,12 @@ import functools import multiprocessing import os import unittest +try: + import fcntl # pylint: disable=import-error,unused-import +except ImportError: + POSIX_MODE = False +else: + POSIX_MODE = True import mock @@ -10,7 +16,6 @@ from certbot import errors from certbot.tests import util as test_util -@test_util.broken_on_windows class LockDirTest(test_util.TempDirTestCase): """Tests for certbot.lock.lock_dir.""" @classmethod @@ -18,6 +23,7 @@ class LockDirTest(test_util.TempDirTestCase): from certbot.lock import lock_dir return lock_dir(*args, **kwargs) + @test_util.broken_on_windows def test_it(self): assert_raises = functools.partial( self.assertRaises, errors.LockError, self._call, self.tempdir) @@ -25,7 +31,6 @@ class LockDirTest(test_util.TempDirTestCase): test_util.lock_and_call(assert_raises, lock_path) -@test_util.broken_on_windows class LockFileTest(test_util.TempDirTestCase): """Tests for certbot.lock.LockFile.""" @classmethod @@ -49,6 +54,7 @@ class LockFileTest(test_util.TempDirTestCase): # Test we're still able to properly acquire and release the lock self.test_removed() + @test_util.broken_on_windows def test_contention(self): assert_raises = functools.partial( self.assertRaises, errors.LockError, self._call, self.lock_path) @@ -71,6 +77,8 @@ class LockFileTest(test_util.TempDirTestCase): self.assertTrue(lock_file.__class__.__name__ in lock_repr) self.assertTrue(self.lock_path in lock_repr) + @test_util.skip_on_windows( + 'Race conditions on lock are specific to the non-blocking file access approach on Linux.') def test_race(self): should_delete = [True, False] stat = os.stat @@ -91,27 +99,36 @@ class LockFileTest(test_util.TempDirTestCase): lock_file.release() self.assertFalse(os.path.exists(self.lock_path)) - @mock.patch('certbot.compat.fcntl.lockf') - def test_unexpected_lockf_err(self, mock_lockf): + def test_unexpected_lockf_or_locking_err(self): + if POSIX_MODE: + mocked_function = 'certbot.lock.fcntl.lockf' + else: + mocked_function = 'certbot.lock.msvcrt.locking' msg = 'hi there' - mock_lockf.side_effect = IOError(msg) - try: - self._call(self.lock_path) - except IOError as err: - self.assertTrue(msg in str(err)) - else: # pragma: no cover - self.fail('IOError not raised') + with mock.patch(mocked_function) as mock_lock: + mock_lock.side_effect = IOError(msg) + try: + self._call(self.lock_path) + except IOError as err: + self.assertTrue(msg in str(err)) + else: # pragma: no cover + self.fail('IOError not raised') - @mock.patch('certbot.lock.os.stat') - def test_unexpected_stat_err(self, mock_stat): + def test_unexpected_os_err(self): + if POSIX_MODE: + mock_function = 'certbot.lock.os.stat' + else: + mock_function = 'certbot.lock.msvcrt.locking' + # The only expected errno are ENOENT and EACCES in lock module. msg = 'hi there' - mock_stat.side_effect = OSError(msg) - try: - self._call(self.lock_path) - except OSError as err: - self.assertTrue(msg in str(err)) - else: # pragma: no cover - self.fail('OSError not raised') + with mock.patch(mock_function) as mock_os: + mock_os.side_effect = OSError(msg) + try: + self._call(self.lock_path) + except OSError as err: + self.assertTrue(msg in str(err)) + else: # pragma: no cover + self.fail('OSError not raised') if __name__ == "__main__": diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 6685b88c6..c85009c62 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -2,7 +2,6 @@ import argparse import errno import os -import shutil import unittest import mock @@ -88,7 +87,6 @@ class LockDirUntilExit(test_util.TempDirTestCase): import certbot.util reload_module(certbot.util) - @test_util.broken_on_windows @mock.patch('certbot.util.logger') @mock.patch('certbot.util.atexit_register') def test_it(self, mock_register, mock_logger): @@ -100,11 +98,15 @@ class LockDirUntilExit(test_util.TempDirTestCase): self.assertEqual(mock_register.call_count, 1) registered_func = mock_register.call_args[0][0] - shutil.rmtree(subdir) - registered_func() # exception not raised - # logger.debug is only called once because the second call - # to lock subdir was ignored because it was already locked - self.assertEqual(mock_logger.debug.call_count, 1) + + from certbot import util + # Despite lock_dir_until_exit has been called twice to subdir, its lock should have been + # added only once. So we expect to have two lock references: for self.tempdir and subdir + self.assertTrue(len(util._LOCKS) == 2) # pylint: disable=protected-access + registered_func() # Exception should not be raised + # Logically, logger.debug, that would be invoked in case of unlock failure, + # should never been called. + self.assertEqual(mock_logger.debug.call_count, 0) class SetUpCoreDirTest(test_util.TempDirTestCase): From 0489ca588851430d03c45fecec56fc1ca8223c83 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 16 Feb 2019 03:51:22 +0100 Subject: [PATCH 05/44] [Windows] Fixes lock_and_call test method (#6772) The method `lock_and_call`, in `certbot.tests.util` is designed to acquire a lock on a foreign process, then execute a callable in the current process. This is done to closely reproduce the lock mechanism involved between two certbot instances that are running in parallel. This method uses the `multiprocessing` module. But its implementation in `lock_and_call` is broken for Windows: the two processes fail to communicate, leading to a deadlock. In fact, `multiprocessing` module is using the fork mechanism on Linux, and the spawn mechanism on Windows, leading to behavior inconsistencies between the two platforms. As this method is for tests, and not for production code, I did not try to make two implementations "by the book", one suitable for Windows, the other for Linux, like for the `certbot.lock` module. Instead, I use a `subprocess` approach with a trigger file allowing to coordinate the current process and the subprocess. With this, `lock_and_call` is running from the same code both on Linux and Windows. Relevant tests in the `certbot.tests.lock_test` test module are now enabled for Windows. * Implement new lock_and_call method * Reactivate tests for Windows --- certbot/tests/lock_test.py | 2 - certbot/tests/util.py | 93 ++++++++++++++++++++++---------------- 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index aa1de299b..8658443d0 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -23,7 +23,6 @@ class LockDirTest(test_util.TempDirTestCase): from certbot.lock import lock_dir return lock_dir(*args, **kwargs) - @test_util.broken_on_windows def test_it(self): assert_raises = functools.partial( self.assertRaises, errors.LockError, self._call, self.tempdir) @@ -54,7 +53,6 @@ class LockFileTest(test_util.TempDirTestCase): # Test we're still able to properly acquire and release the lock self.test_removed() - @test_util.broken_on_windows def test_contention(self): assert_raises = functools.partial( self.assertRaises, errors.LockError, self._call, self.lock_path) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 8c5db2c2f..953d36536 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -3,7 +3,6 @@ .. warning:: This module is not part of the public API. """ -import multiprocessing import os import pkg_resources import shutil @@ -11,6 +10,8 @@ import tempfile import unittest import sys import warnings +import subprocess +import time from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -23,8 +24,8 @@ from six.moves import reload_module # pylint: disable=import-error from certbot import constants from certbot import interfaces from certbot import storage -from certbot import util from certbot import configuration +from certbot import util from certbot.display import util as display_util @@ -211,7 +212,7 @@ class FreezableMock(object): """ def __init__(self, frozen=False, func=None, return_value=mock.sentinel.DEFAULT): - self._frozen_set = set() if frozen else set(('freeze',)) + self._frozen_set = set() if frozen else {'freeze', } self._func = func self._mock = mock.MagicMock() if return_value != mock.sentinel.DEFAULT: @@ -340,6 +341,7 @@ class TempDirTestCase(unittest.TestCase): warnings.warn(message) shutil.rmtree(self.tempdir, onerror=onerror_handler) + class ConfigTestCase(TempDirTestCase): """Test class which sets up a NamespaceConfig object. @@ -358,47 +360,58 @@ class ConfigTestCase(TempDirTestCase): self.config.chain_path = constants.CLI_DEFAULTS['auth_chain_path'] self.config.server = "https://example.com" -def lock_and_call(func, lock_path): - """Grab a lock for lock_path and call func. - - :param callable func: object to call after acquiring the lock - :param str lock_path: path to file or directory to lock +def lock_and_call(callback, path_to_lock): + """Grab a lock on path_to_lock from a foreign process and call the callback. + :param callable callback: object to call after acquiring the lock + :param str path_to_lock: path to file or directory to lock """ - # Reload module to reset internal _LOCKS dictionary + script = """\ +import os +import sys +import time +from certbot import lock + +path_to_lock = sys.argv[1] +trigger = sys.argv[2] + +if os.path.isdir(path_to_lock): + my_lock = lock.lock_dir(path_to_lock) +else: + my_lock = lock.LockFile(path_to_lock) +try: + open(trigger, 'w').close() + while os.path.exists(trigger): + time.sleep(1) +finally: + my_lock.release() +""" + # Reload certbot.util module to reset internal _LOCKS dictionary. reload_module(util) - # start child and wait for it to grab the lock - cv = multiprocessing.Condition() - cv.acquire() - child_args = (cv, lock_path,) - child = multiprocessing.Process(target=hold_lock, args=child_args) - child.start() - cv.wait() + workspace = tempfile.mkdtemp() + try: + tmp_script = os.path.join(workspace, 'test_script.py') + with open(tmp_script, 'w') as file_handle: + file_handle.write(script) - # call func and terminate the child - func() - cv.notify() - cv.release() - child.join() - assert child.exitcode == 0 + # Trigger file is used to coordinate current process and its subprocess. + trigger = os.path.join(workspace, 'trigger') + process = subprocess.Popen([sys.executable, tmp_script, path_to_lock, trigger]) + try: + # Poll and wait for the lock to be acquired, spotted by the trigger file creation. + while not os.path.exists(trigger): + time.sleep(1) + # Then execute the callback. + callback() + finally: + # This will trigger the lock release in subprocess. + os.remove(trigger) + process.communicate() + assert process.returncode == 0 + finally: + shutil.rmtree(workspace) -def hold_lock(cv, lock_path): # pragma: no cover - """Acquire a file lock at lock_path and wait to release it. - - :param multiprocessing.Condition cv: condition for synchronization - :param str lock_path: path to the file lock - - """ - from certbot import lock - if os.path.isdir(lock_path): - my_lock = lock.lock_dir(lock_path) - else: - my_lock = lock.LockFile(lock_path) - cv.acquire() - cv.notify() - cv.wait() - my_lock.release() def skip_on_windows(reason): """Decorator to skip permanently a test on Windows. A reason is required.""" @@ -407,6 +420,7 @@ def skip_on_windows(reason): return unittest.skipIf(sys.platform == 'win32', reason)(function) return wrapper + def broken_on_windows(function): """Decorator to skip temporarily a broken test on Windows.""" reason = 'Test is broken and ignored on windows but should be fixed.' @@ -415,9 +429,10 @@ def broken_on_windows(function): and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true', reason)(function) + def temp_join(path): """ Return the given path joined to the tempdir path for the current platform Eg.: 'cert' => /tmp/cert (Linux) or 'C:\\Users\\currentuser\\AppData\\Temp\\cert' (Windows) """ - return os.path.join(tempfile.gettempdir(), path) + return os.path.join(tempfile.gettempdir(), path) From 209a0c4d2ce6ddd553326e10e71a012b05027bfe Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 20 Feb 2019 00:15:06 +0100 Subject: [PATCH 06/44] [Windows] Refactor lock_and_call using queues (#6778) * Refactor lock_and_call using queues * Update util.py * Replace queue by event * Add comments * Update certbot/tests/util.py Co-Authored-By: adferrand * Update certbot/tests/util.py Co-Authored-By: adferrand * Add control on timeout --- certbot/tests/util.py | 81 ++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 953d36536..289f2dd9b 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -10,8 +10,7 @@ import tempfile import unittest import sys import warnings -import subprocess -import time +from multiprocessing import Process, Event from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -25,6 +24,7 @@ from certbot import constants from certbot import interfaces from certbot import storage from certbot import configuration +from certbot import lock from certbot import util from certbot.display import util as display_util @@ -361,56 +361,49 @@ class ConfigTestCase(TempDirTestCase): self.config.server = "https://example.com" +def _handle_lock(event_in, event_out, path): + """ + Acquire a file lock on given path, then wait to release it. This worker is coordinated + using events to signal when the lock should be acquired and released. + :param multiprocessing.Event event_in: event object to signal when to release the lock + :param multiprocessing.Event event_out: event object to signal when the lock is acquired + :param path: the path to lock + """ + if os.path.isdir(path): + my_lock = lock.lock_dir(path) + else: + my_lock = lock.LockFile(path) + try: + event_out.set() + assert event_in.wait(timeout=20), 'Timeout while waiting to release the lock.' + finally: + my_lock.release() + + def lock_and_call(callback, path_to_lock): - """Grab a lock on path_to_lock from a foreign process and call the callback. + """ + Grab a lock on path_to_lock from a foreign process then execute the callback. :param callable callback: object to call after acquiring the lock :param str path_to_lock: path to file or directory to lock """ - script = """\ -import os -import sys -import time -from certbot import lock - -path_to_lock = sys.argv[1] -trigger = sys.argv[2] - -if os.path.isdir(path_to_lock): - my_lock = lock.lock_dir(path_to_lock) -else: - my_lock = lock.LockFile(path_to_lock) -try: - open(trigger, 'w').close() - while os.path.exists(trigger): - time.sleep(1) -finally: - my_lock.release() -""" # Reload certbot.util module to reset internal _LOCKS dictionary. reload_module(util) - workspace = tempfile.mkdtemp() - try: - tmp_script = os.path.join(workspace, 'test_script.py') - with open(tmp_script, 'w') as file_handle: - file_handle.write(script) + emit_event = Event() + receive_event = Event() + process = Process(target=_handle_lock, args=(emit_event, receive_event, path_to_lock)) + process.start() - # Trigger file is used to coordinate current process and its subprocess. - trigger = os.path.join(workspace, 'trigger') - process = subprocess.Popen([sys.executable, tmp_script, path_to_lock, trigger]) - try: - # Poll and wait for the lock to be acquired, spotted by the trigger file creation. - while not os.path.exists(trigger): - time.sleep(1) - # Then execute the callback. - callback() - finally: - # This will trigger the lock release in subprocess. - os.remove(trigger) - process.communicate() - assert process.returncode == 0 - finally: - shutil.rmtree(workspace) + # Wait confirmation that lock is acquired + assert receive_event.wait(timeout=10), 'Timeout while waiting to acquire the lock.' + # Execute the callback + callback() + # Trigger unlock from foreign process + emit_event.set() + + # Wait for process termination + process.join(timeout=10) + assert process.exitcode == 0 def skip_on_windows(reason): From bda840b3eec938829306956a0b3e7d4f691ddac2 Mon Sep 17 00:00:00 2001 From: sblondon Date: Wed, 20 Feb 2019 01:08:20 +0100 Subject: [PATCH 07/44] add version parameter when the help message is displayed (#6780) --- certbot/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 31f55711f..398124510 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -108,7 +108,7 @@ manage your account with Let's Encrypt: # This is the short help for certbot --help, where we disable argparse # altogether -HELP_USAGE = """ +HELP_AND_VERSION_USAGE = """ More detailed help: -h, --help [TOPIC] print this message, or detailed help on a topic; @@ -117,6 +117,8 @@ More detailed help: all, automation, commands, paths, security, testing, or any of the subcommands or plugins (certonly, renew, install, register, nginx, apache, standalone, webroot, etc.) + + --version print the version number """ @@ -566,7 +568,7 @@ class HelpfulArgumentParser(object): usage = SHORT_USAGE if help_arg == True: - self.notify(usage + COMMAND_OVERVIEW % (apache_doc, nginx_doc) + HELP_USAGE) + self.notify(usage + COMMAND_OVERVIEW % (apache_doc, nginx_doc) + HELP_AND_VERSION_USAGE) sys.exit(0) elif help_arg in self.COMMANDS_TOPICS: self.notify(usage + self._list_subcommands()) From 7c731599a0140868acdb609bc20345f0bd0bd3aa Mon Sep 17 00:00:00 2001 From: sydneyli Date: Fri, 18 Jan 2019 17:09:19 -0800 Subject: [PATCH 08/44] Generate constraints file to pin deps in Docker images Dockerfiles pin versions using constraints file Pulling out strip_hashes and add --no-deps flag --- Dockerfile | 8 +++++++- certbot-dns-cloudflare/Dockerfile | 2 +- certbot-dns-cloudxns/Dockerfile | 2 +- certbot-dns-digitalocean/Dockerfile | 2 +- certbot-dns-dnsimple/Dockerfile | 2 +- certbot-dns-dnsmadeeasy/Dockerfile | 2 +- certbot-dns-gehirn/Dockerfile | 2 +- certbot-dns-google/Dockerfile | 2 +- certbot-dns-linode/Dockerfile | 2 +- certbot-dns-luadns/Dockerfile | 2 +- certbot-dns-nsone/Dockerfile | 2 +- certbot-dns-ovh/Dockerfile | 2 +- certbot-dns-rfc2136/Dockerfile | 2 +- certbot-dns-route53/Dockerfile | 2 +- certbot-dns-sakuracloud/Dockerfile | 2 +- tools/dev_constraints.txt | 2 +- tools/pip_install.py | 6 ++---- tools/strip_hashes.py | 16 ++++++++++++++++ 18 files changed, 40 insertions(+), 20 deletions(-) create mode 100755 tools/strip_hashes.py diff --git a/Dockerfile b/Dockerfile index 8f434e89c..828f5ec94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,13 @@ VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot COPY CHANGELOG.md README.rst setup.py src/ + +# Generate constraints file to pin dependency versions COPY letsencrypt-auto-source/pieces/dependency-requirements.txt . +COPY tools /opt/certbot/tools +RUN sh -c 'cat dependency-requirements.txt | /opt/certbot/tools/strip_hashes.py > unhashed_requirements.txt' +RUN sh -c 'cat tools/dev_constraints.txt unhashed_requirements.txt | /opt/certbot/tools/merge_requirements.py > docker_constraints.txt' + COPY acme src/acme COPY certbot src/certbot @@ -23,7 +29,7 @@ RUN apk add --no-cache --virtual .build-deps \ musl-dev \ libffi-dev \ && pip install -r /opt/certbot/dependency-requirements.txt \ - && pip install --no-cache-dir \ + && pip install --no-cache-dir --no-deps \ --editable /opt/certbot/src/acme \ --editable /opt/certbot/src \ && apk del .build-deps diff --git a/certbot-dns-cloudflare/Dockerfile b/certbot-dns-cloudflare/Dockerfile index 27dcc8751..adbf715fa 100644 --- a/certbot-dns-cloudflare/Dockerfile +++ b/certbot-dns-cloudflare/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-cloudflare -RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudflare diff --git a/certbot-dns-cloudxns/Dockerfile b/certbot-dns-cloudxns/Dockerfile index cc84ea65b..48c88c35c 100644 --- a/certbot-dns-cloudxns/Dockerfile +++ b/certbot-dns-cloudxns/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-cloudxns -RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudxns diff --git a/certbot-dns-digitalocean/Dockerfile b/certbot-dns-digitalocean/Dockerfile index 8bdd0619f..342e0e876 100644 --- a/certbot-dns-digitalocean/Dockerfile +++ b/certbot-dns-digitalocean/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-digitalocean -RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-digitalocean diff --git a/certbot-dns-dnsimple/Dockerfile b/certbot-dns-dnsimple/Dockerfile index 38d2be80e..724675339 100644 --- a/certbot-dns-dnsimple/Dockerfile +++ b/certbot-dns-dnsimple/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-dnsimple -RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsimple diff --git a/certbot-dns-dnsmadeeasy/Dockerfile b/certbot-dns-dnsmadeeasy/Dockerfile index ff7936925..1480baf4f 100644 --- a/certbot-dns-dnsmadeeasy/Dockerfile +++ b/certbot-dns-dnsmadeeasy/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-dnsmadeeasy -RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsmadeeasy diff --git a/certbot-dns-gehirn/Dockerfile b/certbot-dns-gehirn/Dockerfile index 48ad902b5..7dce0e521 100644 --- a/certbot-dns-gehirn/Dockerfile +++ b/certbot-dns-gehirn/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-gehirn -RUN pip install --no-cache-dir --editable src/certbot-dns-gehirn +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-gehirn diff --git a/certbot-dns-google/Dockerfile b/certbot-dns-google/Dockerfile index 4a258d0ee..5750b31d9 100644 --- a/certbot-dns-google/Dockerfile +++ b/certbot-dns-google/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-google -RUN pip install --no-cache-dir --editable src/certbot-dns-google +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-google diff --git a/certbot-dns-linode/Dockerfile b/certbot-dns-linode/Dockerfile index 2e237b521..6db8b59fb 100644 --- a/certbot-dns-linode/Dockerfile +++ b/certbot-dns-linode/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-linode -RUN pip install --no-cache-dir --editable src/certbot-dns-linode +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-linode diff --git a/certbot-dns-luadns/Dockerfile b/certbot-dns-luadns/Dockerfile index 6efb4d777..efc9f36d6 100644 --- a/certbot-dns-luadns/Dockerfile +++ b/certbot-dns-luadns/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-luadns -RUN pip install --no-cache-dir --editable src/certbot-dns-luadns +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-luadns diff --git a/certbot-dns-nsone/Dockerfile b/certbot-dns-nsone/Dockerfile index 88fc13c57..de541e850 100644 --- a/certbot-dns-nsone/Dockerfile +++ b/certbot-dns-nsone/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-nsone -RUN pip install --no-cache-dir --editable src/certbot-dns-nsone +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-nsone diff --git a/certbot-dns-ovh/Dockerfile b/certbot-dns-ovh/Dockerfile index e8da96d95..37e488dc4 100644 --- a/certbot-dns-ovh/Dockerfile +++ b/certbot-dns-ovh/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-ovh -RUN pip install --no-cache-dir --editable src/certbot-dns-ovh +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-ovh diff --git a/certbot-dns-rfc2136/Dockerfile b/certbot-dns-rfc2136/Dockerfile index 1b8feb2f8..3ebb6a72e 100644 --- a/certbot-dns-rfc2136/Dockerfile +++ b/certbot-dns-rfc2136/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-rfc2136 -RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136 +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-rfc2136 diff --git a/certbot-dns-route53/Dockerfile b/certbot-dns-route53/Dockerfile index a1b8d6caf..e1825c11d 100644 --- a/certbot-dns-route53/Dockerfile +++ b/certbot-dns-route53/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-route53 -RUN pip install --no-cache-dir --editable src/certbot-dns-route53 +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-route53 diff --git a/certbot-dns-sakuracloud/Dockerfile b/certbot-dns-sakuracloud/Dockerfile index 694773f61..9fa9b3c22 100644 --- a/certbot-dns-sakuracloud/Dockerfile +++ b/certbot-dns-sakuracloud/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-sakuracloud -RUN pip install --no-cache-dir --editable src/certbot-dns-sakuracloud +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-sakuracloud diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 88340cb00..f9ad7329b 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,4 +1,4 @@ -# Specifies Python package versions for development. +# Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in letsencrypt-auto's requirements file. # Some dev package versions specified here may be overridden by higher level constraints # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). diff --git a/tools/pip_install.py b/tools/pip_install.py index dd6302b48..475799eb1 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -19,6 +19,7 @@ import tempfile import merge_requirements as merge_module import readlink +import strip_hashes def find_tools_path(): @@ -47,10 +48,7 @@ def certbot_normal_processing(tools_path, test_constraints): with open(certbot_requirements, 'r') as fd: data = fd.readlines() with open(test_constraints, 'w') as fd: - for line in data: - search = re.search(r'^(\S*==\S*).*$', line) - if search: - fd.write('{0}{1}'.format(search.group(1), os.linesep)) + fd.write(strip_hashes.main(data)) def merge_requirements(tools_path, requirements, test_constraints, all_constraints): diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py new file mode 100755 index 000000000..c5591e2f4 --- /dev/null +++ b/tools/strip_hashes.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import os +import re +import sys + +def main(args): + out_lines = [] + for line in args: + search = re.search(r'^(\S*==\S*).*$', line) + if search: + out_lines.append(search.group(1)) + return os.linesep.join(out_lines) + +if __name__ == '__main__': + print(main(sys.argv[1:])) From 8bda10541a67fc66c0b4da0ccc5bc8bacf741f25 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 20 Feb 2019 17:00:59 +0200 Subject: [PATCH 09/44] Add stdin option for merge_requirements Add stdin and file path support to strip_hashes --- tools/merge_requirements.py | 43 ++++++++++++++++++++++++++----------- tools/pip_install.py | 3 ++- tools/strip_hashes.py | 39 ++++++++++++++++++++++++++++----- 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index 4205e6bcf..521d28b8f 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -10,27 +10,36 @@ from __future__ import print_function import sys -def read_file(file_path): - """Reads in a Python requirements file. +def process_entries(entries): + """ Ignore empty lines, comments and editable requirements - :param str file_path: path to requirements file + :param list entries: List of entries :returns: mapping from a project to its pinned version :rtype: dict - """ data = {} - with open(file_path) as file_h: - for line in file_h: - line = line.strip() - if line and not line.startswith('#') and not line.startswith('-e'): - project, version = line.split('==') - if not version: - raise ValueError("Unexpected syntax '{0}'".format(line)) - data[project] = version + for e in entries: + e = e.strip() + if e and not e.startswith('#') and not e.startswith('-e'): + project, version = e.split('==') + if not version: + raise ValueError("Unexpected syntax '{0}'".format(e)) + data[project] = version return data +def read_file(file_path): + """Reads in a Python requirements file. + + :param str file_path: path to requirements file + + :returns: list of entries in the file + :rtype: list + + """ + with open(file_path) as file_h: + return file_h.readlines() def output_requirements(requirements): """Prepare print requirements to stdout. @@ -53,7 +62,15 @@ def main(*paths): """ data = {} for path in paths: - data.update(read_file(path)) + data.update(process_entries(read_file(path))) + + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + data.update(process_entries(stdin_data)) + return output_requirements(data) diff --git a/tools/pip_install.py b/tools/pip_install.py index 475799eb1..8f0437d9c 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -48,7 +48,8 @@ def certbot_normal_processing(tools_path, test_constraints): with open(certbot_requirements, 'r') as fd: data = fd.readlines() with open(test_constraints, 'w') as fd: - fd.write(strip_hashes.main(data)) + data = os.linesep.join(strip_hashes.process_entries(data)) + fd.write(data) def merge_requirements(tools_path, requirements, test_constraints, all_constraints): diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py index c5591e2f4..7e7809458 100755 --- a/tools/strip_hashes.py +++ b/tools/strip_hashes.py @@ -1,16 +1,45 @@ #!/usr/bin/env python +"""Removes hash information from requirement files passed to it as file path +arguments or simply piped to stdin.""" import os import re import sys -def main(args): + +def process_entries(entries): + """Strips off hash strings from dependencies. + + :param list entries: List of entries + + :returns: list of dependencies without hashes + :rtype: list + """ out_lines = [] - for line in args: - search = re.search(r'^(\S*==\S*).*$', line) + for e in entries: + e = e.strip() + search = re.search(r'^(\S*==\S*).*$', e) if search: out_lines.append(search.group(1)) - return os.linesep.join(out_lines) + return out_lines + +def main(*paths): + """Reads dependency definitions from a (list of) file(s) or stdin and + removes hashes from returned entries""" + + deps = [] + for path in paths: + with open(path) as file_h: + deps += process_entries(file_h.readlines()) + + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + deps += process_entries(stdin_data) + + return os.linesep.join(deps) if __name__ == '__main__': - print(main(sys.argv[1:])) + print(main(*sys.argv[1:])) # pylint: disable=star-args From eef4c476335a9338dc71509d3d7c21de7d81b486 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 20 Feb 2019 15:20:44 -0800 Subject: [PATCH 10/44] Add failure message if test farm tests do not run the correct number of tests. (#6771) Fixes #6748. --- tests/letstest/multitester.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 8babc67b3..b8ae937ad 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -564,6 +564,11 @@ try: ii, target, status = outq print('%d %s %s'%(ii, target['name'], status)) results_file.write('%d %s %s\n'%(ii, target['name'], status)) + if len(outputs) != num_processes: + failure_message = 'FAILURE: Some target machines failed to run and were not tested. ' +\ + 'Tests should be rerun.' + print(failure_message) + results_file.write(failure_message + '\n') results_file.close() finally: From eb5c4eca877baf52c0c39778002fce1e9482cff7 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 21 Feb 2019 01:20:16 +0100 Subject: [PATCH 11/44] [Windows] Working unit tests for certbot-nginx (#6782) This PR fixes certbot-nginx and relevant tests to make them succeed on Windows. Next step will be to enable integration tests through certbot-ci in a future PR. * Fix tests and incompabilities in certbot-nginx for Windows * Fix lint, fix oldest local dependencies --- certbot-nginx/certbot_nginx/configurator.py | 7 +++---- certbot-nginx/certbot_nginx/parser.py | 4 ++-- .../certbot_nginx/tests/configurator_test.py | 19 ++++++++----------- .../certbot_nginx/tests/http_01_test.py | 6 ------ .../certbot_nginx/tests/parser_test.py | 12 +++++++++--- .../certbot_nginx/tests/tls_sni_01_test.py | 6 ------ certbot-nginx/certbot_nginx/tests/util.py | 18 ++++++++++++++++++ certbot-nginx/local-oldest-requirements.txt | 4 ++-- tools/install_and_test.py | 2 +- tox.cover.py | 5 ++++- 10 files changed, 47 insertions(+), 36 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index dd0bf9e8b..ffe1ddac7 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -13,6 +13,7 @@ import zope.interface from acme import challenges from acme import crypto_util as acme_crypto_util +from certbot import compat from certbot import constants as core_constants from certbot import crypto_util from certbot import errors @@ -164,9 +165,7 @@ class NginxConfigurator(common.Installer): util.lock_dir_until_exit(self.conf('server-root')) except (OSError, errors.LockError): logger.debug('Encountered error:', exc_info=True) - raise errors.PluginError( - 'Unable to lock %s', self.conf('server-root')) - + raise errors.PluginError('Unable to lock {0}'.format(self.conf('server-root'))) # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, @@ -899,7 +898,7 @@ class NginxConfigurator(common.Installer): have permissions of root. """ - uid = os.geteuid() + uid = compat.os_geteuid() util.make_or_verify_dir( self.config.work_dir, core_constants.CONFIG_DIRS_MODE, uid) util.make_or_verify_dir( diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 622eb8d55..c5f780d94 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -81,9 +81,9 @@ class NginxParser(object): """ if not os.path.isabs(path): - return os.path.join(self.root, path) + return os.path.normpath(os.path.join(self.root, path)) else: - return path + return os.path.normpath(path) def _build_addr_to_ssl(self): """Builds a map from address to whether it listens on ssl in any server block diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 957588e2a..706e2637a 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -1,7 +1,6 @@ # pylint: disable=too-many-public-methods """Test for certbot_nginx.configurator.""" import os -import shutil import unittest import mock @@ -33,12 +32,6 @@ class NginxConfiguratorTest(util.NginxTest): self.config = util.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - shutil.rmtree(self.logs_dir) - @mock.patch("certbot_nginx.configurator.util.exe_exists") def test_prepare_no_install(self, mock_exe_exists): mock_exe_exists.return_value = False @@ -69,8 +62,11 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare_locked(self): server_root = self.config.conf("server-root") + + from certbot import util as certbot_util + certbot_util._LOCKS[server_root].release() # pylint: disable=protected-access + self.config.config_test = mock.Mock() - os.remove(os.path.join(server_root, ".certbot.lock")) certbot_test_util.lock_and_call(self._test_prepare_locked, server_root) @mock.patch("certbot_nginx.configurator.util.exe_exists") @@ -88,11 +84,11 @@ class NginxConfiguratorTest(util.NginxTest): def test_get_all_names(self, mock_gethostbyaddr): mock_gethostbyaddr.return_value = ('155.225.50.69.nephoscale.net', [], []) names = self.config.get_all_names() - self.assertEqual(names, set( - ["155.225.50.69.nephoscale.net", "www.example.org", "another.alias", + self.assertEqual(names, { + "155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", - "headers.com"])) + "headers.com"}) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -171,6 +167,7 @@ class NginxConfiguratorTest(util.NginxTest): 'abc.www.foo.com': "etc_nginx/foo.conf", 'www.bar.co.uk': "etc_nginx/nginx.conf", 'ipv6.com': "etc_nginx/sites-enabled/ipv6.com"} + conf_path = {key: os.path.normpath(value) for key, value in conf_path.items()} vhost = self.config.choose_vhosts(name)[0] path = os.path.relpath(vhost.filep, self.temp_dir) diff --git a/certbot-nginx/certbot_nginx/tests/http_01_test.py b/certbot-nginx/certbot_nginx/tests/http_01_test.py index ed3c257ee..41c4b95fc 100644 --- a/certbot-nginx/certbot_nginx/tests/http_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/http_01_test.py @@ -1,6 +1,5 @@ """Tests for certbot_nginx.http_01""" import unittest -import shutil import mock import six @@ -54,11 +53,6 @@ class HttpPerformTest(util.NginxTest): from certbot_nginx import http_01 self.http01 = http_01.NginxHttp01(config) - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - def test_perform0(self): responses = self.http01.perform() self.assertEqual([], responses) diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index f6f28e42b..3a68f7f24 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -67,9 +67,15 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods def test_abs_path(self): nparser = parser.NginxParser(self.config_path) - self.assertEqual('/etc/nginx/*', nparser.abs_path('/etc/nginx/*')) - self.assertEqual(os.path.join(self.config_path, 'foo/bar/'), - nparser.abs_path('foo/bar/')) + if os.name != 'nt': + self.assertEqual('/etc/nginx/*', nparser.abs_path('/etc/nginx/*')) + self.assertEqual(os.path.join(self.config_path, 'foo/bar'), + nparser.abs_path('foo/bar')) + else: + self.assertEqual('C:\\etc\\nginx\\*', nparser.abs_path('C:\\etc\\nginx\\*')) + self.assertEqual(os.path.join(self.config_path, 'foo\\bar'), + nparser.abs_path('foo\\bar')) + def test_filedump(self): nparser = parser.NginxParser(self.config_path) diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 72b65911c..62ca085ef 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -1,6 +1,5 @@ """Tests for certbot_nginx.tls_sni_01""" import unittest -import shutil import mock import six @@ -55,11 +54,6 @@ class TlsSniPerformTest(util.NginxTest): from certbot_nginx import tls_sni_01 self.sni = tls_sni_01.NginxTlsSni01(config) - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - @mock.patch("certbot_nginx.configurator" ".NginxConfigurator.choose_vhosts") def test_perform(self, mock_choose): diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index ad1af2b96..ef669dac0 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -4,6 +4,8 @@ import os import pkg_resources import tempfile import unittest +import shutil +import warnings import josepy as jose import mock @@ -33,6 +35,22 @@ class NginxTest(unittest.TestCase): # pylint: disable=too-few-public-methods self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector( "rsa512_key.pem")) + def tearDown(self): + # On Windows we have various files which are not correctly closed at the time of tearDown. + # For know, we log them until a proper file close handling is written. + # Useful for development only, so no warning when we are on a CI process. + def onerror_handler(_, path, excinfo): + """On error handler""" + if not os.environ.get('APPVEYOR'): # pragma: no cover + message = ('Following error occurred when deleting path {0}' + 'during tearDown process: {1}'.format(path, str(excinfo))) + warnings.warn(message) + + shutil.rmtree(self.temp_dir, onerror=onerror_handler) + shutil.rmtree(self.config_dir, onerror=onerror_handler) + shutil.rmtree(self.work_dir, onerror=onerror_handler) + shutil.rmtree(self.logs_dir, onerror=onerror_handler) + def get_data_filename(filename): """Gets the filename of a test data file.""" diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index bcd02d197..db6b261f0 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.26.0 -certbot[dev]==0.22.0 +acme[dev]==0.29.0 +-e .[dev] diff --git a/tools/install_and_test.py b/tools/install_and_test.py index b15c8eca5..288226527 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -15,7 +15,7 @@ import subprocess import re SKIP_PROJECTS_ON_WINDOWS = [ - 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + 'certbot-apache', 'certbot-postfix', 'letshelp-certbot'] def call_with_print(command, cwd=None): diff --git a/tox.cover.py b/tox.cover.py index 008424641..e323ba255 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -35,7 +35,8 @@ COVER_THRESHOLDS = { } SKIP_PROJECTS_ON_WINDOWS = [ - 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + 'certbot-apache', 'certbot-postfix', 'letshelp-certbot'] + def cover(package): threshold = COVER_THRESHOLDS.get(package)['windows' if os.name == 'nt' else 'linux'] @@ -54,6 +55,7 @@ def cover(package): sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', '{0}/*'.format(pkg_dir), '--show-missing']) + def main(): description = """ This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order @@ -77,5 +79,6 @@ Option -e makes sure we fail fast and don't submit to codecov.""" for package in packages: cover(package) + if __name__ == '__main__': main() From b10ceb7d907cdcc36ec2d745e7802127919ec043 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 22 Feb 2019 01:55:08 +0100 Subject: [PATCH 12/44] Fix test sdists with atexit handlers (#6769) So merging the study from @bmw and me, here is what happened. Each invocation of `certbot.logger.post_arg_parse_setup` create a file handler on `letsencrypt.log`. This function also set an atexit handler invoking `logger.shutdown()`, that have the effect to close all logger file handler not already closed at this point. This method is supposed to be called when a python process is close to exit, because it makes all logger unable to write new logs on any handler. Before #6667 and this PR, for tests, the atexit handle would be triggered only at the end of the pytest process. It means that each test that launches `certbot.logger.post_arg_parse_setup` add a new file handler. These tests were typically connecting the file handler on a `letsencrypt.log` located in a temporary directory, and this directory and content was wipped out at each test tearDown. As a consequence, the file handles, not cleared from the logger, were accumulating in the logger, with all of them connected to a deleted file log, except the last one that was just created by the current test. Considering the number of tests concerned, there were ~300 file handler at the end of pytest execution. One can see that, on prior #6667, by calling `print(logger.getLogger().handlers` on the `tearDown` of these tests, and see the array growing at each test execution. Even if this represent a memory leak, this situation was not really a problem on Linux: because a file can be deleted before it is closed, it was only meaning that a given invocation of `logger.debug` for instance, during the tests, was written in 300 log files. The overhead is negligeable. On Windows however, the file handlers were failing because you cannot delete a file before it is closed. It was one of the reason for #6667, that added a call to `logging.shutdown()` at each test tearDown, with the consequence to close all file handlers. At this point, Linux is not happy anymore. Any call to `logger.warn` will generate an error for each closed file handler. As a file handler is added for each test, the number of errors grows on each test, following an arithmetical suite divergence. On `test_sdists.py`, that is using the bare setuptools test suite without output capturing, we can see the damages. The total output takes 216000 lines, and 23000 errors are generated. A decent machine can support this load, but a not a small AWS instance, that is crashing during the execution. Even with pytest, the captured output and the memory leak become so large that segfaults are generated. On the current PR, the problem is solved, by resetting the file handlers array on the logging system on each test tearDown. So each fileHandler is properly closed, and removed from the stack. They do not participate anymore in the logging system, and can be garbage collected. Then we stay on always one file handler opened at any time, and tests can succeed on AWS instances. For the record, here is all the places where the logging system is called and fail if there is still file handlers closed but not cleaned (extracted from the original huge output before correction): ``` Logged from file account.py, line 116 Logged from file account.py, line 178 Logged from file client.py, line 166 Logged from file client.py, line 295 Logged from file client.py, line 415 Logged from file client.py, line 422 Logged from file client.py, line 480 Logged from file client.py, line 503 Logged from file client.py, line 540 Logged from file client.py, line 601 Logged from file client.py, line 622 Logged from file client.py, line 750 Logged from file cli.py, line 220 Logged from file cli.py, line 226 Logged from file crypto_util.py, line 101 Logged from file crypto_util.py, line 127 Logged from file crypto_util.py, line 147 Logged from file crypto_util.py, line 261 Logged from file crypto_util.py, line 283 Logged from file crypto_util.py, line 307 Logged from file crypto_util.py, line 336 Logged from file disco.py, line 116 Logged from file disco.py, line 124 Logged from file disco.py, line 134 Logged from file disco.py, line 138 Logged from file disco.py, line 141 Logged from file dns_common_lexicon.py, line 45 Logged from file dns_common_lexicon.py, line 61 Logged from file dns_common_lexicon.py, line 67 Logged from file dns_common.py, line 316 Logged from file dns_common.py, line 64 Logged from file eff.py, line 60 Logged from file eff.py, line 73 Logged from file error_handler.py, line 105 Logged from file error_handler.py, line 110 Logged from file error_handler.py, line 87 Logged from file hooks.py, line 248 Logged from file main.py, line 1071 Logged from file main.py, line 1075 Logged from file main.py, line 1189 Logged from file ops.py, line 122 Logged from file ops.py, line 325 Logged from file ops.py, line 338 Logged from file reporter.py, line 55 Logged from file selection.py, line 110 Logged from file selection.py, line 118 Logged from file selection.py, line 123 Logged from file selection.py, line 176 Logged from file selection.py, line 231 Logged from file selection.py, line 310 Logged from file selection.py, line 66 Logged from file standalone.py, line 101 Logged from file standalone.py, line 88 Logged from file standalone.py, line 97 Logged from file standalone.py, line 98 Logged from file storage.py, line 52 Logged from file storage.py, line 59 Logged from file storage.py, line 75 Logged from file util.py, line 56 Logged from file webroot.py, line 165 Logged from file webroot.py, line 186 Logged from file webroot.py, line 187 Logged from file webroot.py, line 204 Logged from file webroot.py, line 223 Logged from file webroot.py, line 234 Logged from file webroot.py, line 235 Logged from file webroot.py, line 237 Logged from file webroot.py, line 91 ``` * Reapply #6667 * Make setuptools delegates tests execution to pytest, like in acme module. * Clean handlers at each tearDown to avoid memory leaks. * Update changelog --- CHANGELOG.md | 2 ++ acme/setup.py | 4 +++- certbot-apache/setup.py | 20 ++++++++++++++++++++ certbot-nginx/setup.py | 20 ++++++++++++++++++++ certbot/tests/lock_test.py | 3 +++ certbot/tests/util.py | 33 ++++++++++++++++++--------------- certbot/tests/util_test.py | 24 +++++++++--------------- certbot/util.py | 6 +++--- setup.py | 20 ++++++++++++++++++++ 9 files changed, 98 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57a895e07..7744e3745 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). warnings described at https://github.com/certbot/josepy/issues/13. * Apache plugin now respects CERTBOT_DOCS environment variable when adding command line defaults. +* Tests execution for certbot, certbot-apache and certbot-nginx packages now relies on pytest. ### Fixed @@ -26,6 +27,7 @@ package with changes other than its version number was: * acme * certbot * certbot-apache +* certbot-nginx More details about these changes can be found on our GitHub repo. diff --git a/acme/setup.py b/acme/setup.py index 79d6d3389..6cb5c3f92 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -36,6 +36,7 @@ docs_extras = [ 'sphinx_rtd_theme', ] + class PyTest(TestCommand): user_options = [] @@ -50,6 +51,7 @@ class PyTest(TestCommand): errno = pytest.main(shlex.split(self.pytest_args)) sys.exit(errno) + setup( name='acme', version=version, @@ -82,7 +84,7 @@ setup( 'dev': dev_extras, 'docs': docs_extras, }, - tests_require=["pytest"], test_suite='acme', + tests_require=["pytest"], cmdclass={"test": PyTest}, ) diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 52528a536..5d15611dd 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,5 +1,7 @@ from setuptools import setup from setuptools import find_packages +from setuptools.command.test import test as TestCommand +import sys version = '0.32.0.dev0' @@ -21,6 +23,22 @@ docs_extras = [ 'sphinx_rtd_theme', ] + +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + + setup( name='certbot-apache', version=version, @@ -64,4 +82,6 @@ setup( ], }, test_suite='certbot_apache', + tests_require=["pytest"], + cmdclass={"test": PyTest}, ) diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index f78473ada..8984704a6 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,5 +1,7 @@ from setuptools import setup from setuptools import find_packages +from setuptools.command.test import test as TestCommand +import sys version = '0.32.0.dev0' @@ -21,6 +23,22 @@ docs_extras = [ 'sphinx_rtd_theme', ] + +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + + setup( name='certbot-nginx', version=version, @@ -64,4 +82,6 @@ setup( ], }, test_suite='certbot_nginx', + tests_require=["pytest"], + cmdclass={"test": PyTest}, ) diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 8658443d0..6379693ae 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -41,6 +41,7 @@ class LockFileTest(test_util.TempDirTestCase): super(LockFileTest, self).setUp() self.lock_path = os.path.join(self.tempdir, 'test.lock') + @test_util.broken_on_windows def test_acquire_without_deletion(self): # acquire the lock in another process but don't delete the file child = multiprocessing.Process(target=self._call, @@ -58,6 +59,7 @@ class LockFileTest(test_util.TempDirTestCase): self.assertRaises, errors.LockError, self._call, self.lock_path) test_util.lock_and_call(assert_raises, self.lock_path) + @test_util.broken_on_windows def test_locked_repr(self): lock_file = self._call(self.lock_path) locked_repr = repr(lock_file) @@ -92,6 +94,7 @@ class LockFileTest(test_util.TempDirTestCase): self._call(self.lock_path) self.assertFalse(should_delete) + @test_util.broken_on_windows def test_removed(self): lock_file = self._call(self.lock_path) lock_file.release() diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 289f2dd9b..4c0c66a42 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -3,13 +3,14 @@ .. warning:: This module is not part of the public API. """ +import logging import os import pkg_resources import shutil +import stat import tempfile import unittest import sys -import warnings from multiprocessing import Process, Event from cryptography.hazmat.backends import default_backend @@ -329,23 +330,25 @@ class TempDirTestCase(unittest.TestCase): def tearDown(self): """Execute after test""" - # On Windows we have various files which are not correctly closed at the time of tearDown. - # For know, we log them until a proper file close handling is written. - # Useful for development only, so no warning when we are on a CI process. - def onerror_handler(_, path, excinfo): - """On error handler""" - if not os.environ.get('APPVEYOR'): # pragma: no cover - message = ('Following error occurred when deleting the tempdir {0}' - ' for path {1} during tearDown process: {2}' - .format(self.tempdir, path, str(excinfo))) - warnings.warn(message) - shutil.rmtree(self.tempdir, onerror=onerror_handler) + # Cleanup opened resources after a test. This is usually done through atexit handlers in + # Certbot, but during tests, atexit will not run registered functions before tearDown is + # called and instead will run them right before the entire test process exits. + # It is a problem on Windows, that does not accept to clean resources before closing them. + logging.shutdown() + # Remove logging handlers that have been closed so they won't be + # accidentally used in future tests. + logging.getLogger().handlers = [] + util._release_locks() # pylint: disable=protected-access + + def handle_rw_files(_, path, __): + """Handle read-only files, that will fail to be removed on Windows.""" + os.chmod(path, stat.S_IWRITE) + os.remove(path) + shutil.rmtree(self.tempdir, onerror=handle_rw_files) class ConfigTestCase(TempDirTestCase): - """Test class which sets up a NamespaceConfig object. - - """ + """Test class which sets up a NamespaceConfig object.""" def setUp(self): super(ConfigTestCase, self).setUp() self.config = configuration.NamespaceConfig( diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index c85009c62..cac587995 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -193,7 +193,12 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_wrong_mode(self): os.chmod(self.tempdir, 0o400) - self.assertFalse(self._call(0o600)) + try: + self.assertFalse(self._call(0o600)) + finally: + # Without proper write permissions, Windows is unable to delete a folder, + # even with admin permissions. Write access must be explicitly set first. + os.chmod(self.tempdir, 0o700) class UniqueFileTest(test_util.TempDirTestCase): @@ -279,20 +284,9 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): for f, _ in items: f.close() - @mock.patch("certbot.util.os.fdopen") - def test_failure(self, mock_fdopen): - err = OSError("whoops") - err.errno = errno.EIO - mock_fdopen.side_effect = err - self.assertRaises(OSError, self._call, "wow") - - @mock.patch("certbot.util.os.fdopen") - def test_subsequent_failure(self, mock_fdopen): - self._call("wow") - err = OSError("whoops") - err.errno = errno.EIO - mock_fdopen.side_effect = err - self.assertRaises(OSError, self._call, "wow") + def test_failure(self): + with mock.patch("certbot.util.os.open", side_effect=OSError(errno.EIO)): + self.assertRaises(OSError, self._call, "wow") class SafelyRemoveTest(test_util.TempDirTestCase): diff --git a/certbot/util.py b/certbot/util.py index d7c542465..416075ce8 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -142,6 +142,7 @@ def _release_locks(): except: # pylint: disable=bare-except msg = 'Exception occurred releasing lock: {0!r}'.format(dir_lock) logger.debug(msg, exc_info=True) + _LOCKS.clear() def set_up_core_dir(directory, mode, uid, strict): @@ -225,9 +226,8 @@ def safe_open(path, mode="w", chmod=None, buffering=None): fdopen_args = () # type: Union[Tuple[()], Tuple[int]] if buffering is not None: fdopen_args = (buffering,) - return os.fdopen( - os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args), - mode, *fdopen_args) + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args) + return os.fdopen(fd, mode, *fdopen_args) def _unique_file(path, filename_pat, count, chmod, mode): diff --git a/setup.py b/setup.py index 14fef37f3..26a1c4293 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,10 @@ import codecs import os import re +import sys from setuptools import find_packages, setup +from setuptools.command.test import test as TestCommand # Workaround for http://bugs.python.org/issue8876, see # http://bugs.python.org/issue8876#msg208792 @@ -77,6 +79,22 @@ docs_extras = [ 'sphinx_rtd_theme', ] + +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + + setup( name='certbot', version=version, @@ -123,6 +141,8 @@ setup( # to test all packages run "python setup.py test -s # {acme,certbot_apache,certbot_nginx}" test_suite='certbot', + tests_require=["pytest"], + cmdclass={"test": PyTest}, entry_points={ 'console_scripts': [ From 31b4b8e57c542577d35a20f87eaad60d5ce37193 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 22 Feb 2019 06:42:01 -0800 Subject: [PATCH 13/44] Log the execution of manual hooks (#6788) * Move logging to execute and fix tests. * update changelog --- CHANGELOG.md | 2 ++ certbot/hooks.py | 36 ++++++++++++++++++++---------------- certbot/plugins/manual.py | 7 +++++-- certbot/tests/hook_test.py | 34 +++++++++++++++++++--------------- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7744e3745..ce7eac2e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). warnings described at https://github.com/certbot/josepy/issues/13. * Apache plugin now respects CERTBOT_DOCS environment variable when adding command line defaults. +* The running of manual plugin hooks is now always included in Certbot's log + output. * Tests execution for certbot, certbot-apache and certbot-nginx packages now relies on pytest. ### Fixed diff --git a/certbot/hooks.py b/certbot/hooks.py index d5239a437..7d2e42fcd 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -93,8 +93,7 @@ def _run_pre_hook_if_necessary(command): if command in executed_pre_hooks: logger.info("Pre-hook command already run, skipping: %s", command) else: - logger.info("Running pre-hook command: %s", command) - _run_hook(command) + _run_hook("pre-hook", command) executed_pre_hooks.add(command) @@ -126,8 +125,7 @@ def post_hook(config): _run_eventually(cmd) # certonly / run elif cmd: - logger.info("Running post-hook command: %s", cmd) - _run_hook(cmd) + _run_hook("post-hook", cmd) post_hooks = [] # type: List[str] @@ -149,8 +147,7 @@ def _run_eventually(command): def run_saved_post_hooks(): """Run any post hooks that were saved up in the course of the 'renew' verb""" for cmd in post_hooks: - logger.info("Running post-hook command: %s", cmd) - _run_hook(cmd) + _run_hook("post-hook", cmd) def deploy_hook(config, domains, lineage_path): @@ -220,23 +217,30 @@ def _run_deploy_hook(command, domains, lineage_path, dry_run): os.environ["RENEWED_DOMAINS"] = " ".join(domains) os.environ["RENEWED_LINEAGE"] = lineage_path - logger.info("Running deploy-hook command: %s", command) - _run_hook(command) + _run_hook("deploy-hook", command) -def _run_hook(shell_cmd): +def _run_hook(cmd_name, shell_cmd): """Run a hook command. - :returns: stderr if there was any""" + :param str cmd_name: the user facing name of the hook being run + :param shell_cmd: shell command to execute + :type shell_cmd: `list` of `str` or `str` - err, _ = execute(shell_cmd) + :returns: stderr if there was any""" + err, _ = execute(cmd_name, shell_cmd) return err -def execute(shell_cmd): +def execute(cmd_name, shell_cmd): """Run a command. + :param str cmd_name: the user facing name of the hook being run + :param shell_cmd: shell command to execute + :type shell_cmd: `list` of `str` or `str` + :returns: `tuple` (`str` stderr, `str` stdout)""" + logger.info("Running %s command: %s", cmd_name, shell_cmd) # universal_newlines causes Popen.communicate() # to return str objects instead of bytes in Python 3 @@ -245,12 +249,12 @@ def execute(shell_cmd): out, err = cmd.communicate() base_cmd = os.path.basename(shell_cmd.split(None, 1)[0]) if out: - logger.info('Output from %s:\n%s', base_cmd, out) + logger.info('Output from %s command %s:\n%s', cmd_name, base_cmd, out) if cmd.returncode != 0: - logger.error('Hook command "%s" returned error code %d', - shell_cmd, cmd.returncode) + logger.error('%s command "%s" returned error code %d', + cmd_name, shell_cmd, cmd.returncode) if err: - logger.error('Error output from %s:\n%s', base_cmd, err) + logger.error('Error output from %s command %s:\n%s', cmd_name, base_cmd, err) return (err, out) diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 8723a1c62..123a5bfea 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -202,7 +202,7 @@ permitted by DNS standards.) os.environ.pop('CERTBOT_KEY_PATH', None) os.environ.pop('CERTBOT_SNI_DOMAIN', None) os.environ.update(env) - _, out = hooks.execute(self.conf('auth-hook')) + _, out = self._execute_hook('auth-hook') env['CERTBOT_AUTH_OUTPUT'] = out.strip() self.env[achall] = env @@ -243,5 +243,8 @@ permitted by DNS standards.) if 'CERTBOT_TOKEN' not in env: os.environ.pop('CERTBOT_TOKEN', None) os.environ.update(env) - hooks.execute(self.conf('cleanup-hook')) + self._execute_hook('cleanup-hook') self.reverter.recovery_routine() + + def _execute_hook(self, hook_name): + return hooks.execute(self.option_name(hook_name), self.conf(hook_name)) diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index f5bb0c8b5..90f639958 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -121,7 +121,7 @@ class PreHookTest(HookTest): def _test_nonrenew_common(self): mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.config.pre_hook) + mock_execute.assert_called_once_with("pre-hook", self.config.pre_hook) self._test_no_executions_common() def test_no_hooks(self): @@ -137,21 +137,21 @@ class PreHookTest(HookTest): def test_renew_disabled_dir_hooks(self): self.config.directory_hooks = False mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.config.pre_hook) + mock_execute.assert_called_once_with("pre-hook", self.config.pre_hook) self._test_no_executions_common() def test_renew_no_overlap(self): self.config.verb = "renew" mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_any_call(self.dir_hook) - mock_execute.assert_called_with(self.config.pre_hook) + mock_execute.assert_any_call("pre-hook", self.dir_hook) + mock_execute.assert_called_with("pre-hook", self.config.pre_hook) self._test_no_executions_common() def test_renew_with_overlap(self): self.config.pre_hook = self.dir_hook self.config.verb = "renew" mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.dir_hook) + mock_execute.assert_called_once_with("pre-hook", self.dir_hook) self._test_no_executions_common() def _test_no_executions_common(self): @@ -193,7 +193,7 @@ class PostHookTest(HookTest): for verb in ("certonly", "run",): self.config.verb = verb mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.config.post_hook) + mock_execute.assert_called_once_with("post-hook", self.config.post_hook) self.assertFalse(self._get_eventually()) def test_cert_only_and_run_without_hook(self): @@ -277,12 +277,12 @@ class RunSavedPostHooksTest(HookTest): calls = mock_execute.call_args_list for actual_call, expected_arg in zip(calls, self.eventually): - self.assertEqual(actual_call[0][0], expected_arg) + self.assertEqual(actual_call[0][1], expected_arg) def test_single(self): self.eventually = ["foo"] mock_execute = self._call_with_mock_execute_and_eventually() - mock_execute.assert_called_once_with(self.eventually[0]) + mock_execute.assert_called_once_with("post-hook", self.eventually[0]) class RenewalHookTest(HookTest): @@ -360,7 +360,7 @@ class DeployHookTest(RenewalHookTest): self.config.deploy_hook = "foo" mock_execute = self._call_with_mock_execute( self.config, domains, lineage) - mock_execute.assert_called_once_with(self.config.deploy_hook) + mock_execute.assert_called_once_with("deploy-hook", self.config.deploy_hook) class RenewHookTest(RenewalHookTest): @@ -384,7 +384,7 @@ class RenewHookTest(RenewalHookTest): self.config.directory_hooks = False mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - mock_execute.assert_called_once_with(self.config.renew_hook) + mock_execute.assert_called_once_with("deploy-hook", self.config.renew_hook) @mock.patch("certbot.hooks.logger") def test_dry_run(self, mock_logger): @@ -408,13 +408,13 @@ class RenewHookTest(RenewalHookTest): self.config.renew_hook = self.dir_hook mock_execute = self._call_with_mock_execute( self.config, ["example.net", "example.org"], "/foo/bar") - mock_execute.assert_called_once_with(self.dir_hook) + mock_execute.assert_called_once_with("deploy-hook", self.dir_hook) def test_no_overlap(self): mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - mock_execute.assert_any_call(self.dir_hook) - mock_execute.assert_called_with(self.config.renew_hook) + mock_execute.assert_any_call("deploy-hook", self.dir_hook) + mock_execute.assert_called_with("deploy-hook", self.config.renew_hook) class ExecuteTest(unittest.TestCase): @@ -433,18 +433,22 @@ class ExecuteTest(unittest.TestCase): def _test_common(self, returncode, stdout, stderr): given_command = "foo" + given_name = "foo-hook" with mock.patch("certbot.hooks.Popen") as mock_popen: mock_popen.return_value.communicate.return_value = (stdout, stderr) mock_popen.return_value.returncode = returncode with mock.patch("certbot.hooks.logger") as mock_logger: - self.assertEqual(self._call(given_command), (stderr, stdout)) + self.assertEqual(self._call(given_name, given_command), (stderr, stdout)) executed_command = mock_popen.call_args[1].get( "args", mock_popen.call_args[0][0]) self.assertEqual(executed_command, given_command) + mock_logger.info.assert_any_call("Running %s command: %s", + given_name, given_command) if stdout: - self.assertTrue(mock_logger.info.called) + mock_logger.info.assert_any_call(mock.ANY, mock.ANY, + mock.ANY, stdout) if stderr or returncode: self.assertTrue(mock_logger.error.called) From f105aedc9258d7a1de55b7de56802535d3b1b4dc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 22 Feb 2019 16:55:50 -0800 Subject: [PATCH 14/44] Remove display.py. --- tests/display.py | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 tests/display.py diff --git a/tests/display.py b/tests/display.py deleted file mode 100644 index 1f548e33d..000000000 --- a/tests/display.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Manual test of display functions.""" -import sys - -from certbot.display import util -from certbot.tests.display import util_test - - -def test_visual(displayer, choices): - """Visually test all of the display functions.""" - displayer.notification("Random notification!") - displayer.menu("Question?", choices, - ok_label="O", cancel_label="Can", help_label="??") - displayer.menu("Question?", [choice[1] for choice in choices], - ok_label="O", cancel_label="Can", help_label="??") - displayer.input("Input Message") - displayer.yesno("YesNo Message", yes_label="Yessir", no_label="Nosir") - displayer.checklist("Checklist Message", [choice[0] for choice in choices]) - - -if __name__ == "__main__": - displayer = util.FileDisplay(sys.stdout, False) - test_visual(displayer, util_test.CHOICES) From 401045be89d241bad01ebb8e933bdb83f836bc20 Mon Sep 17 00:00:00 2001 From: Julie R Date: Sat, 23 Feb 2019 03:02:43 +0100 Subject: [PATCH 15/44] Add acme library usage example (http-01) (#5494) * Add acme library usage example Create, edit and deactivate account. Setup and perform http-01 challenge. Issue, renew and revoke certificate. * Adapt example to ACME-v2 and exclude data persistence The code to persist/load data would length this example and distract from what is actually important. * Fix domain names and e-mail addresses * Remove unnecessary license header This usage example is under the license for the acme package. * Remove logging information The code will be mostly read by developers, so simplify the logging info into comments. * Revert abstraction of simple methods All methods that are used only once in this example were expanded into the main code in order to make the process more explicit. * Fix missing URL suffix * Improve aesthetics and reorganize workflow Also make words capitalization consistent and improve comments. No complaints from pep8. --- acme/examples/http01_example.py | 240 ++++++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 acme/examples/http01_example.py diff --git a/acme/examples/http01_example.py b/acme/examples/http01_example.py new file mode 100644 index 000000000..79508f1b4 --- /dev/null +++ b/acme/examples/http01_example.py @@ -0,0 +1,240 @@ +"""Example ACME-V2 API for HTTP-01 challenge. + +Brief: + +This a complete usage example of the python-acme API. + +Limitations of this example: + - Works for only one Domain name + - Performs only HTTP-01 challenge + - Uses ACME-v2 + +Workflow: + (Account creation) + - Create account key + - Register account and accept TOS + (Certificate actions) + - Select HTTP-01 within offered challenges by the CA server + - Set up http challenge resource + - Set up standalone web server + - Create domain private key and CSR + - Issue certificate + - Renew certificate + - Revoke certificate + (Account update actions) + - Change contact information + - Deactivate Account +""" +from contextlib import contextmanager +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +import OpenSSL + +from acme import challenges +from acme import client +from acme import crypto_util +from acme import errors +from acme import messages +from acme import standalone +import josepy as jose + +# Constants: + +# This is the staging point for ACME-V2 within Let's Encrypt. +DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory' + +USER_AGENT = 'python-acme-example' + +# Account key size +ACC_KEY_BITS = 2048 + +# Certificate private key size +CERT_PKEY_BITS = 2048 + +# Domain name for the certificate. +DOMAIN = 'client.example.com' + +# If you are running Boulder locally, it is possible to configure any port +# number to execute the challenge, but real CA servers will always use port +# 80, as described in the ACME specification. +PORT = 80 + + +# Useful methods and classes: + + +def new_csr_comp(domain_name, pkey_pem=None): + """Create certificate signing request.""" + if pkey_pem is None: + # Create private key. + pkey = OpenSSL.crypto.PKey() + pkey.generate_key(OpenSSL.crypto.TYPE_RSA, CERT_PKEY_BITS) + pkey_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, + pkey) + csr_pem = crypto_util.make_csr(pkey_pem, [domain_name]) + return pkey_pem, csr_pem + + +def select_http01_chall(orderr): + """Extract authorization resource from within order resource.""" + # Authorization Resource: authz. + # This object holds the offered challenges by the server and their status. + authz_list = orderr.authorizations + + for authz in authz_list: + # Choosing challenge. + # authz.body.challenges is a set of ChallengeBody objects. + for i in authz.body.challenges: + # Find the supported challenge. + if isinstance(i.chall, challenges.HTTP01): + return i + + raise Exception('HTTP-01 challenge was not offered by the CA server.') + + +@contextmanager +def challenge_server(http_01_resources): + """Manage standalone server set up and shutdown.""" + + # Setting up a fake server that binds at PORT and any address. + address = ('', PORT) + try: + servers = standalone.HTTP01DualNetworkedServers(address, + http_01_resources) + # Start client standalone web server. + servers.serve_forever() + yield servers + finally: + # Shutdown client web server and unbind from PORT + servers.shutdown_and_server_close() + + +def perform_http01(client_acme, challb, orderr): + """Set up standalone webserver and perform HTTP-01 challenge.""" + + response, validation = challb.response_and_validation(client_acme.net.key) + + resource = standalone.HTTP01RequestHandler.HTTP01Resource( + chall=challb.chall, response=response, validation=validation) + + with challenge_server({resource}): + # Let the CA server know that we are ready for the challenge. + client_acme.answer_challenge(challb, response) + + # Wait for challenge status and then issue a certificate. + # It is possible to set a deadline time. + finalized_orderr = client_acme.poll_and_finalize(orderr) + + return finalized_orderr.fullchain_pem + + +# Main examples: + + +def example_http(): + """This example executes the whole process of fulfilling a HTTP-01 + challenge for one specific domain. + + The workflow consists of: + (Account creation) + - Create account key + - Register account and accept TOS + (Certificate actions) + - Select HTTP-01 within offered challenges by the CA server + - Set up http challenge resource + - Set up standalone web server + - Create domain private key and CSR + - Issue certificate + - Renew certificate + - Revoke certificate + (Account update actions) + - Change contact information + - Deactivate Account + + """ + # Create account key + + acc_key = jose.JWKRSA( + key=rsa.generate_private_key(public_exponent=65537, + key_size=ACC_KEY_BITS, + backend=default_backend())) + + # Register account and accept TOS + + net = client.ClientNetwork(acc_key, user_agent=USER_AGENT) + directory = messages.Directory.from_json(net.get(DIRECTORY_URL).json()) + client_acme = client.ClientV2(directory, net=net) + + # Terms of Service URL is in client_acme.directory.meta.terms_of_service + # Registration Resource: regr + # Creates account with contact information. + email = ('fake@example.com') + regr = client_acme.new_account( + messages.NewRegistration.from_data( + email=email, terms_of_service_agreed=True)) + + # Create domain private key and CSR + pkey_pem, csr_pem = new_csr_comp(DOMAIN) + + # Issue certificate + + orderr = client_acme.new_order(csr_pem) + + # Select HTTP-01 within offered challenges by the CA server + challb = select_http01_chall(orderr) + + # The certificate is ready to be used in the variable "fullchain_pem". + fullchain_pem = perform_http01(client_acme, challb, orderr) + + # Renew certificate + + _, csr_pem = new_csr_comp(DOMAIN, pkey_pem) + + orderr = client_acme.new_order(csr_pem) + + challb = select_http01_chall(orderr) + + # Performing challenge + fullchain_pem = perform_http01(client_acme, challb, orderr) + + # Revoke certificate + + fullchain_com = jose.ComparableX509( + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + + try: + client_acme.revoke(fullchain_com, 0) # revocation reason = 0 + except errors.ConflictError: + # Certificate already revoked. + pass + + # Query registration status. + client_acme.net.account = regr + try: + regr = client_acme.query_registration(regr) + except errors.Error as err: + if err.typ == messages.OLD_ERROR_PREFIX + 'unauthorized' \ + or err.typ == messages.ERROR_PREFIX + 'unauthorized': + # Status is deactivated. + pass + raise + + # Change contact information + + email = 'newfake@example.com' + regr = client_acme.update_registration( + regr.update( + body=regr.body.update( + contact=('mailto:' + email,) + ) + ) + ) + + # Deactivate account/registration + + regr = client_acme.deactivate_registration(regr) + + +if __name__ == "__main__": + example_http() From 339d034d6a5a57d296607795a4706203f81d7059 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 27 Feb 2019 18:21:47 +0100 Subject: [PATCH 16/44] Remove keyAuthorization field from the challenge response JWS token (#6758) Fixes #6755. POSTing the `keyAuthorization` in a JWS token when answering an ACME challenge, has been deprecated for some time now. Indeed, this is superfluous as the request is already authentified by the JWS signature. Boulder still accepts to see this field in the JWS token, and ignore it. Pebble in non strict mode also. But Pebble in strict mode refuses the request, to prepare complete removal of this field in ACME v2. Certbot still sends the `keyAuthorization` field. This PR removes it, and makes Certbot compliant with current ACME v2 protocol, and so Pebble in strict mode. See also [letsencrypt/pebble#192](https://github.com/letsencrypt/pebble/issues/192) for implementation details server side. * New implementation, with a fallback. * Add deprecation on changelog * Update acme/acme/client.py Co-Authored-By: adferrand * Fix an instance parameter * Update changelog, extend coverage * Update comment * Add unit tests on keyAuthorization dump * Update acme/acme/client.py Co-Authored-By: adferrand * Restrict the magic of setting a variable in immutable object in one place. Make a soon to be removed method private. --- CHANGELOG.md | 7 +++++++ acme/acme/challenges.py | 20 ++++++++++++++++++++ acme/acme/challenges_test.py | 12 ++++++++++++ acme/acme/client.py | 26 ++++++++++++++++++++------ acme/acme/client_test.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce7eac2e9..53c158f8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,13 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * The running of manual plugin hooks is now always included in Certbot's log output. * Tests execution for certbot, certbot-apache and certbot-nginx packages now relies on pytest. +* The `acme` module avoids sending the `keyAuthorization` field in the JWS + payload when responding to a challenge as the field is not included in the + current ACME protocol. To ease the migration path for ACME CA servers, + Certbot and its `acme` module will first try the request without the + `keyAuthorization` field but will temporarily retry the request with the + field included if a `malformed` error is received. This fallback will be + removed in version 0.34.0. ### Fixed diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 501f74881..6f2b3757b 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -108,6 +108,10 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): key_authorization = jose.Field("keyAuthorization") thumbprint_hash_function = hashes.SHA256 + def __init__(self, *args, **kwargs): + super(KeyAuthorizationChallengeResponse, self).__init__(*args, **kwargs) + self._dump_authorization_key(False) + def verify(self, chall, account_public_key): """Verify the key authorization. @@ -140,6 +144,22 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return True + def _dump_authorization_key(self, dump): + # type: (bool) -> None + """ + Set if keyAuthorization is dumped in the JSON representation of this ChallengeResponse. + NB: This method is declared as private because it will eventually be removed. + :param bool dump: True to dump the keyAuthorization, False otherwise + """ + object.__setattr__(self, '_dump_auth_key', dump) + + def to_partial_json(self): + jobj = super(KeyAuthorizationChallengeResponse, self).to_partial_json() + if not self._dump_auth_key: # pylint: disable=no-member + jobj.pop('keyAuthorization', None) + + return jobj + @six.add_metaclass(abc.ABCMeta) class KeyAuthorizationChallenge(_TokenChallenge): diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 81d39058e..4b905c1e5 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -94,6 +94,9 @@ class DNS01ResponseTest(unittest.TestCase): self.response = self.chall.response(KEY) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.msg.to_partial_json()) + self.msg._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.msg.to_partial_json()) def test_from_json(self): @@ -165,6 +168,9 @@ class HTTP01ResponseTest(unittest.TestCase): self.response = self.chall.response(KEY) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.msg.to_partial_json()) + self.msg._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.msg.to_partial_json()) def test_from_json(self): @@ -285,6 +291,9 @@ class TLSSNI01ResponseTest(unittest.TestCase): self.assertEqual(self.z_domain, self.response.z_domain) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.response.to_partial_json()) + self.response._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.response.to_partial_json()) def test_from_json(self): @@ -419,6 +428,9 @@ class TLSALPN01ResponseTest(unittest.TestCase): self.response = self.chall.response(KEY) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.msg.to_partial_json()) + self.msg._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.msg.to_partial_json()) def test_from_json(self): diff --git a/acme/acme/client.py b/acme/acme/client.py index 76b6a7788..d74713a7e 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -17,6 +17,7 @@ import requests from requests.adapters import HTTPAdapter import sys +from acme import challenges from acme import crypto_util from acme import errors from acme import jws @@ -155,7 +156,23 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self._post(challb.uri, response) + # Because sending keyAuthorization in a response challenge has been removed from the ACME + # spec, it is not included in the KeyAuthorizationResponseChallenge JSON by default. + # However as a migration path, we temporarily expect a malformed error from the server, + # and fallback by resending the challenge response with the keyAuthorization field. + # TODO: Remove this fallback for Certbot 0.34.0 + try: + response = self._post(challb.uri, response) + except messages.Error as error: + if (error.code == 'malformed' + and isinstance(response, challenges.KeyAuthorizationChallengeResponse)): + logger.debug('Error while responding to a challenge without keyAuthorization ' + 'in the JWS, your ACME CA server may not support it:\n%s', error) + logger.debug('Retrying request with keyAuthorization set.') + response._dump_authorization_key(True) # pylint: disable=protected-access + response = self._post(challb.uri, response) + else: + raise try: authzr_uri = response.links['up']['url'] except KeyError: @@ -781,7 +798,7 @@ class ClientV2(ClientBase): except messages.Error as error: if error.code == 'malformed': logger.debug('Error during a POST-as-GET request, ' - 'your ACME CA may not support it:\n%s', error) + 'your ACME CA server may not support it:\n%s', error) logger.debug('Retrying request with GET.') else: # pragma: no cover raise @@ -1191,10 +1208,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, acme_version=1, **kwargs): - try: - new_nonce_url = kwargs.pop('new_nonce_url') - except KeyError: - new_nonce_url = None + new_nonce_url = kwargs.pop('new_nonce_url', None) data = self._wrap_in_jws(obj, self._get_nonce(url, new_nonce_url), url, acme_version) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index b3d0f1921..dc8cc0c93 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -463,6 +463,34 @@ class ClientTest(ClientTestBase): errors.ClientError, self.client.answer_challenge, self.challr.body, challenges.DNSResponse(validation=None)) + def test_answer_challenge_key_authorization_fallback(self): + self.response.links['up'] = {'url': self.challr.authzr_uri} + self.response.json.return_value = self.challr.body.to_json() + + def _wrapper_post(url, obj, *args, **kwargs): # pylint: disable=unused-argument + """ + Simulate an old ACME CA server, that would respond a 'malformed' + error if keyAuthorization is missing. + """ + jobj = obj.to_partial_json() + if 'keyAuthorization' not in jobj: + raise messages.Error.with_code('malformed') + return self.response + self.net.post.side_effect = _wrapper_post + + # This challenge response is of type KeyAuthorizationChallengeResponse, so the fallback + # should be triggered, and avoid an exception. + http_chall_response = challenges.HTTP01Response(key_authorization='test', + resource=mock.MagicMock()) + self.client.answer_challenge(self.challr.body, http_chall_response) + + # This challenge response is not of type KeyAuthorizationChallengeResponse, so the fallback + # should not be triggered, leading to an exception. + dns_chall_response = challenges.DNSResponse(validation=None) + self.assertRaises( + errors.Error, self.client.answer_challenge, + self.challr.body, dns_chall_response) + def test_retry_after_date(self): self.response.headers['Retry-After'] = 'Fri, 31 Dec 1999 23:59:59 GMT' self.assertEqual( From 9c405a3cd12588ba27cdd8f79c8a101831bd6ad3 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 28 Feb 2019 00:16:52 +0100 Subject: [PATCH 17/44] Fix cryptography OCSP support (#6751) * Reenabling OCSP cryptography support * Refactor the validation logic of OCSP response to match the OpenSSL one * Prepare runtime for OCSP response test * Move unrelated test to another relevant place * Reimplement OCSP status checks in integration tests * Clean script * Protect OCSP check against connection errors * Update tests/certbot-boulder-integration.sh Co-Authored-By: adferrand * Cleaning * Add a specific script for letsencrypt-auto install+help * Remove inconsistent assertion * Add executable permissions * Remove unused variable * Move testdata * Corrected cleanup code * Empty commit --- CHANGELOG.md | 3 + certbot/crypto_util.py | 53 +++-- certbot/ocsp.py | 211 ++++++++++++++---- certbot/tests/ocsp_test.py | 172 ++++++++++++-- certbot/tests/testdata/google_certificate.pem | 41 ++++ .../testdata/google_issuer_certificate.pem | 26 +++ certbot/util.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 40 ++-- .../pieces/dependency-requirements.txt | 40 ++-- tests/certbot-boulder-integration.sh | 55 +++++ .../meta.json | 0 .../private_key.json | 0 .../regr.json | 0 .../a.encryption-example.com/cert1.pem | 0 .../a.encryption-example.com/chain1.pem | 0 .../a.encryption-example.com/fullchain1.pem | 0 .../a.encryption-example.com/privkey1.pem | 0 .../b.encryption-example.com/cert1.pem | 0 .../b.encryption-example.com/chain1.pem | 0 .../b.encryption-example.com/fullchain1.pem | 0 .../b.encryption-example.com/privkey1.pem | 0 .../sample-config/csr/0000_csr-certbot.pem | 0 .../sample-config/csr/0001_csr-certbot.pem | 0 .../sample-config/csr/0002_csr-certbot.pem | 0 .../sample-config/csr/0003_csr-certbot.pem | 0 .../sample-config/keys/0000_key-certbot.pem | 0 .../sample-config/keys/0001_key-certbot.pem | 0 .../sample-config/keys/0002_key-certbot.pem | 0 .../sample-config/keys/0003_key-certbot.pem | 0 .../live/a.encryption-example.com/README | 0 .../live/a.encryption-example.com/cert.pem | 0 .../live/a.encryption-example.com/chain.pem | 0 .../a.encryption-example.com/fullchain.pem | 0 .../live/a.encryption-example.com/privkey.pem | 0 .../live/b.encryption-example.com/README | 0 .../live/b.encryption-example.com/cert.pem | 0 .../live/b.encryption-example.com/chain.pem | 0 .../b.encryption-example.com/fullchain.pem | 0 .../live/b.encryption-example.com/privkey.pem | 0 .../sample-config/options-ssl-apache.conf | 0 .../renewal/a.encryption-example.com.conf | 0 .../renewal/b.encryption-example.com.conf | 0 ...st_letsencrypt_auto_certonly_standalone.sh | 21 -- 43 files changed, 519 insertions(+), 145 deletions(-) create mode 100644 certbot/tests/testdata/google_certificate.pem create mode 100644 certbot/tests/testdata/google_issuer_certificate.pem rename tests/{letstest/testdata => integration}/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json (100%) rename tests/{letstest/testdata => integration}/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json (100%) rename tests/{letstest/testdata => integration}/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/a.encryption-example.com/cert1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/a.encryption-example.com/chain1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/a.encryption-example.com/fullchain1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/a.encryption-example.com/privkey1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/b.encryption-example.com/cert1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/b.encryption-example.com/chain1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/b.encryption-example.com/fullchain1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/archive/b.encryption-example.com/privkey1.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/csr/0000_csr-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/csr/0001_csr-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/csr/0002_csr-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/csr/0003_csr-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/keys/0000_key-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/keys/0001_key-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/keys/0002_key-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/keys/0003_key-certbot.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/a.encryption-example.com/README (100%) rename tests/{letstest/testdata => integration}/sample-config/live/a.encryption-example.com/cert.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/a.encryption-example.com/chain.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/a.encryption-example.com/fullchain.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/a.encryption-example.com/privkey.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/b.encryption-example.com/README (100%) rename tests/{letstest/testdata => integration}/sample-config/live/b.encryption-example.com/cert.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/b.encryption-example.com/chain.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/b.encryption-example.com/fullchain.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/live/b.encryption-example.com/privkey.pem (100%) rename tests/{letstest/testdata => integration}/sample-config/options-ssl-apache.conf (100%) rename tests/{letstest/testdata => integration}/sample-config/renewal/a.encryption-example.com.conf (100%) rename tests/{letstest/testdata => integration}/sample-config/renewal/b.encryption-example.com.conf (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53c158f8f..3eea1923d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,9 @@ More details about these changes can be found on our GitHub repo. * Avoid reprocessing challenges that are already validated when a certificate is issued. +* If possible, Certbot uses built-in support for OCSP from recent cryptography + versions instead of the OpenSSL binary: as a consequence Certbot does not need + the OpenSSL binary to be installed anymore if cryptography>=2.5 is installed. * Support for initiating (but not solving end-to-end) TLS-ALPN-01 challenges with the `acme` module. diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index c4a389cd5..66c68eb38 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -19,7 +19,7 @@ from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey # https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore @@ -226,7 +226,7 @@ def verify_renewable_cert(renewable_cert): def verify_renewable_cert_sig(renewable_cert): - """ Verifies the signature of a `.storage.RenewableCert` object. + """Verifies the signature of a `.storage.RenewableCert` object. :param `.storage.RenewableCert` renewable_cert: cert to verify @@ -239,22 +239,8 @@ def verify_renewable_cert_sig(renewable_cert): cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) pk = chain.public_key() with warnings.catch_warnings(): - warnings.simplefilter("ignore") - if isinstance(pk, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = pk.verifier( # type: ignore - cert.signature, PKCS1v15(), cert.signature_hash_algorithm - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - elif isinstance(pk, EllipticCurvePublicKey): - verifier = pk.verifier( - cert.signature, ECDSA(cert.signature_hash_algorithm) - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - else: - raise errors.Error("Unsupported public key type") + verify_signed_payload(pk, cert.signature, cert.tbs_certificate_bytes, + cert.signature_hash_algorithm) except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) @@ -262,6 +248,37 @@ def verify_renewable_cert_sig(renewable_cert): raise errors.Error(error_str) +def verify_signed_payload(public_key, signature, payload, signature_hash_algorithm): + """Check the signature of a payload. + + :param RSAPublicKey/EllipticCurvePublicKey public_key: the public_key to check signature + :param bytes signature: the signature bytes + :param bytes payload: the payload bytes + :param cryptography.hazmat.primitives.hashes.HashAlgorithm + signature_hash_algorithm: algorithm used to hash the payload + + :raises InvalidSignature: If signature verification fails. + :raises errors.Error: If public key type is not supported + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + if isinstance(public_key, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = public_key.verifier( # type: ignore + signature, PKCS1v15(), signature_hash_algorithm + ) + verifier.update(payload) + verifier.verify() + elif isinstance(public_key, EllipticCurvePublicKey): + verifier = public_key.verifier( + signature, ECDSA(signature_hash_algorithm) + ) + verifier.update(payload) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") + + def verify_cert_matches_priv_key(cert_path, key_path): """ Verifies that the private key and cert match. diff --git a/certbot/ocsp.py b/certbot/ocsp.py index 049e14827..6f4c0b0fb 100644 --- a/certbot/ocsp.py +++ b/certbot/ocsp.py @@ -1,53 +1,79 @@ """Tools for checking certificate revocation.""" import logging import re - +from datetime import datetime, timedelta from subprocess import Popen, PIPE +try: + # Only cryptography>=2.5 has ocsp module + # and signature_hash_algorithm attribute in OCSPResponse class + from cryptography.x509 import ocsp # pylint: disable=import-error + getattr(ocsp.OCSPResponse, 'signature_hash_algorithm') +except (ImportError, AttributeError): # pragma: no cover + ocsp = None # type: ignore +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature +import requests + +from acme.magic_typing import Optional, Tuple # pylint: disable=unused-import, no-name-in-module +from certbot import crypto_util from certbot import errors from certbot import util logger = logging.getLogger(__name__) + class RevocationChecker(object): - "This class figures out OCSP checking on this system, and performs it." + """This class figures out OCSP checking on this system, and performs it.""" - def __init__(self): + def __init__(self, enforce_openssl_binary_usage=False): self.broken = False + self.use_openssl_binary = enforce_openssl_binary_usage or not ocsp - if not util.exe_exists("openssl"): - logger.info("openssl not installed, can't check revocation") - self.broken = True - return - - # New versions of openssl want -header var=val, old ones want -header var val - test_host_format = Popen(["openssl", "ocsp", "-header", "var", "val"], - stdout=PIPE, stderr=PIPE, universal_newlines=True) - _out, err = test_host_format.communicate() - if "Missing =" in err: - self.host_args = lambda host: ["Host=" + host] - else: - self.host_args = lambda host: ["Host", host] + if self.use_openssl_binary: + if not util.exe_exists("openssl"): + logger.info("openssl not installed, can't check revocation") + self.broken = True + return + # New versions of openssl want -header var=val, old ones want -header var val + test_host_format = Popen(["openssl", "ocsp", "-header", "var", "val"], + stdout=PIPE, stderr=PIPE, universal_newlines=True) + _out, err = test_host_format.communicate() + if "Missing =" in err: + self.host_args = lambda host: ["Host=" + host] + else: + self.host_args = lambda host: ["Host", host] def ocsp_revoked(self, cert_path, chain_path): + # type: (str, str) -> bool """Get revoked status for a particular cert version. .. todo:: Make this a non-blocking call :param str cert_path: Path to certificate :param str chain_path: Path to intermediate cert - :rtype bool or None: :returns: True if revoked; False if valid or the check failed + :rtype: bool """ if self.broken: return False - - url, host = self.determine_ocsp_server(cert_path) - if not host: + url, host = _determine_ocsp_server(cert_path) + if not host or not url: return False + + if self.use_openssl_binary: + return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url) + else: + return _check_ocsp_cryptography(cert_path, chain_path, url) + + def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url): + # type: (str, str, str, str) -> bool # jdkasten thanks "Bulletproof SSL and TLS - Ivan Ristic" for documenting this! cmd = ["openssl", "ocsp", "-no_nonce", @@ -65,33 +91,131 @@ class RevocationChecker(object): except errors.SubprocessError: logger.info("OCSP check failed for %s (are we offline?)", cert_path) return False - return _translate_ocsp_query(cert_path, output, err) - def determine_ocsp_server(self, cert_path): - """Extract the OCSP server host from a certificate. +def _determine_ocsp_server(cert_path): + # type: (str) -> Tuple[Optional[str], Optional[str]] + """Extract the OCSP server host from a certificate. - :param str cert_path: Path to the cert we're checking OCSP for - :rtype tuple: - :returns: (OCSP server URL or None, OCSP server host or None) + :param str cert_path: Path to the cert we're checking OCSP for + :rtype tuple: + :returns: (OCSP server URL or None, OCSP server host or None) - """ - try: - url, _err = util.run_script( - ["openssl", "x509", "-in", cert_path, "-noout", "-ocsp_uri"], - log=logger.debug) - except errors.SubprocessError: - logger.info("Cannot extract OCSP URI from %s", cert_path) - return None, None + """ + with open(cert_path, 'rb') as file_handler: + cert = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + try: + extension = cert.extensions.get_extension_for_class(x509.AuthorityInformationAccess) + ocsp_oid = x509.AuthorityInformationAccessOID.OCSP + descriptions = [description for description in extension.value + if description.access_method == ocsp_oid] + + url = descriptions[0].access_location.value + except (x509.ExtensionNotFound, IndexError): + logger.info("Cannot extract OCSP URI from %s", cert_path) + return None, None + + url = url.rstrip() + host = url.partition("://")[2].rstrip("/") + + if host: + return url, host + else: + logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) + return None, None + + +def _check_ocsp_cryptography(cert_path, chain_path, url): + # type: (str, str, str) -> bool + # Retrieve OCSP response + with open(chain_path, 'rb') as file_handler: + issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + with open(cert_path, 'rb') as file_handler: + cert = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + builder = ocsp.OCSPRequestBuilder() + builder = builder.add_certificate(cert, issuer, hashes.SHA1()) + request = builder.build() + request_binary = request.public_bytes(serialization.Encoding.DER) + try: + response = requests.post(url, data=request_binary, + headers={'Content-Type': 'application/ocsp-request'}) + except requests.exceptions.RequestException: + logger.info("OCSP check failed for %s (are we offline?)", cert_path, exc_info=True) + return False + if response.status_code != 200: + logger.info("OCSP check failed for %s (HTTP status: %d)", cert_path, response.status_code) + return False + + response_ocsp = ocsp.load_der_ocsp_response(response.content) + + # Check OCSP response validity + if response_ocsp.response_status != ocsp.OCSPResponseStatus.SUCCESSFUL: + logger.error("Invalid OCSP response status for %s: %s", + cert_path, response_ocsp.response_status) + return False + + # Check OCSP signature + try: + _check_ocsp_response(response_ocsp, request, issuer) + except UnsupportedAlgorithm as e: + logger.error(str(e)) + except errors.Error as e: + logger.error(str(e)) + except InvalidSignature: + logger.error('Invalid signature on OCSP response for %s', cert_path) + except AssertionError as error: + logger.error('Invalid OCSP response for %s: %s.', cert_path, str(error)) + else: + # Check OCSP certificate status + logger.debug("OCSP certificate status for %s is: %s", + cert_path, response_ocsp.certificate_status) + return response_ocsp.certificate_status == ocsp.OCSPCertStatus.REVOKED + + return False + + +def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert): + """Verify that the OCSP is valid for serveral criterias""" + # Assert OCSP response corresponds to the certificate we are talking about + if response_ocsp.serial_number != request_ocsp.serial_number: + raise AssertionError('the certificate in response does not correspond ' + 'to the certificate in request') + + # Assert signature is valid + _check_ocsp_response_signature(response_ocsp, issuer_cert) + + # Assert issuer in response is the expected one + if (not isinstance(response_ocsp.hash_algorithm, type(request_ocsp.hash_algorithm)) + or response_ocsp.issuer_key_hash != request_ocsp.issuer_key_hash + or response_ocsp.issuer_name_hash != request_ocsp.issuer_name_hash): + raise AssertionError('the issuer does not correspond to issuer of the certificate.') + + # In following checks, two situations can occur: + # * nextUpdate is set, and requirement is thisUpdate < now < nextUpdate + # * nextUpdate is not set, and requirement is thisUpdate < now + # NB1: We add a validity period tolerance to handle clock time inconsistencies, + # value is 5 min like for OpenSSL. + # NB2: Another check is to verify that thisUpdate is not too old, it is optional + # for OpenSSL, so we do not do it here. + # See OpenSSL implementation as a reference: + # https://github.com/openssl/openssl/blob/ef45aa14c5af024fcb8bef1c9007f3d1c115bd85/crypto/ocsp/ocsp_cl.c#L338-L391 + now = datetime.now() + if not response_ocsp.this_update: + raise AssertionError('param thisUpdate is not set.') + if response_ocsp.this_update > now + timedelta(minutes=5): + raise AssertionError('param thisUpdate is in the future.') + if response_ocsp.next_update and response_ocsp.next_update < now - timedelta(minutes=5): + raise AssertionError('param nextUpdate is in the past.') + + +def _check_ocsp_response_signature(response_ocsp, issuer_cert): + """Verify an OCSP response signature against certificate issuer""" + # Following line may raise UnsupportedAlgorithm + chosen_hash = response_ocsp.signature_hash_algorithm + crypto_util.verify_signed_payload(issuer_cert.public_key(), response_ocsp.signature, + response_ocsp.tbs_response_bytes, chosen_hash) - url = url.rstrip() - host = url.partition("://")[2].rstrip("/") - if host: - return url, host - else: - logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) - return None, None def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): """Parse openssl's weird output to work out what it means.""" @@ -102,7 +226,7 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): warning = good.group(1) if good else None - if (not "Response verify OK" in ocsp_errors) or (good and warning) or unknown: + if ("Response verify OK" not in ocsp_errors) or (good and warning) or unknown: logger.info("Revocation status for %s is unknown", cert_path) logger.debug("Uncertain output:\n%s\nstderr:\n%s", ocsp_output, ocsp_errors) return False @@ -115,6 +239,5 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): return True else: logger.warning("Unable to properly parse OCSP output: %s\nstderr:%s", - ocsp_output, ocsp_errors) + ocsp_output, ocsp_errors) return False - diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 55cd24adb..ad3467e5a 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -1,18 +1,33 @@ """Tests for ocsp.py""" # pylint: disable=protected-access - import unittest +from datetime import datetime, timedelta +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature +from cryptography import x509 +try: + # Only cryptography>=2.5 has ocsp module + # and signature_hash_algorithm attribute in OCSPResponse class + from cryptography.x509 import ocsp as ocsp_lib # pylint: disable=import-error + getattr(ocsp_lib.OCSPResponse, 'signature_hash_algorithm') +except (ImportError, AttributeError): # pragma: no cover + ocsp_lib = None # type: ignore import mock from certbot import errors +from certbot.tests import util as test_util out = """Missing = in header key=value ocsp: Use -help for summary. """ -class OCSPTest(unittest.TestCase): +class OCSPTestOpenSSL(unittest.TestCase): + """ + OCSP revokation tests using OpenSSL binary. + """ def setUp(self): from certbot import ocsp @@ -22,7 +37,7 @@ class OCSPTest(unittest.TestCase): mock_communicate.communicate.return_value = (None, out) mock_popen.return_value = mock_communicate mock_exists.return_value = True - self.checker = ocsp.RevocationChecker() + self.checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) def tearDown(self): pass @@ -37,23 +52,23 @@ class OCSPTest(unittest.TestCase): mock_exists.return_value = True from certbot import ocsp - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 1) self.assertEqual(checker.host_args("x"), ["Host=x"]) mock_communicate.communicate.return_value = (None, out.partition("\n")[2]) - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(checker.host_args("x"), ["Host", "x"]) self.assertEqual(checker.broken, False) mock_exists.return_value = False mock_popen.call_count = 0 - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 0) self.assertEqual(mock_log.call_count, 1) self.assertEqual(checker.broken, True) - @mock.patch('certbot.ocsp.RevocationChecker.determine_ocsp_server') + @mock.patch('certbot.ocsp._determine_ocsp_server') @mock.patch('certbot.util.run_script') def test_ocsp_revoked(self, mock_run, mock_determine): self.checker.broken = True @@ -71,21 +86,12 @@ class OCSPTest(unittest.TestCase): self.assertEqual(self.checker.ocsp_revoked("x", "y"), False) self.assertEqual(mock_run.call_count, 2) + def test_determine_ocsp_server(self): + cert_path = test_util.vector_path('google_certificate.pem') - @mock.patch('certbot.ocsp.logger.info') - @mock.patch('certbot.util.run_script') - def test_determine_ocsp_server(self, mock_run, mock_info): - uri = "http://ocsp.stg-int-x1.letsencrypt.org/" - host = "ocsp.stg-int-x1.letsencrypt.org" - mock_run.return_value = uri, "" - self.assertEqual(self.checker.determine_ocsp_server("beep"), (uri, host)) - mock_run.return_value = "ftp:/" + host + "/", "" - self.assertEqual(self.checker.determine_ocsp_server("beep"), (None, None)) - self.assertEqual(mock_info.call_count, 1) - - c = "confusion" - mock_run.side_effect = errors.SubprocessError(c) - self.assertEqual(self.checker.determine_ocsp_server("beep"), (None, None)) + from certbot import ocsp + result = ocsp._determine_ocsp_server(cert_path) + self.assertEqual(('http://ocsp.digicert.com', 'ocsp.digicert.com'), result) @mock.patch('certbot.ocsp.logger') @mock.patch('certbot.util.run_script') @@ -112,6 +118,129 @@ class OCSPTest(unittest.TestCase): self.assertEqual(mock_log.info.call_count, 1) +@unittest.skipIf(not ocsp_lib, + reason='This class tests functionalities available only on cryptography>=2.5.0') +class OSCPTestCryptography(unittest.TestCase): + """ + OCSP revokation tests using Cryptography >= 2.4.0 + """ + + def setUp(self): + from certbot import ocsp + self.checker = ocsp.RevocationChecker() + self.cert_path = test_util.vector_path('google_certificate.pem') + self.chain_path = test_util.vector_path('google_issuer_certificate.pem') + + @mock.patch('certbot.ocsp._determine_ocsp_server') + @mock.patch('certbot.ocsp._check_ocsp_cryptography') + def test_ensure_cryptography_toggled(self, mock_revoke, mock_determine): + mock_determine.return_value = ('http://example.com', 'example.com') + self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + mock_revoke.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com') + + @mock.patch('certbot.ocsp.requests.post') + @mock.patch('certbot.ocsp.ocsp.load_der_ocsp_response') + def test_revoke(self, mock_ocsp_response, mock_post): + with mock.patch('certbot.ocsp.crypto_util.verify_signed_payload'): + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertTrue(revoked) + + @mock.patch('certbot.ocsp.crypto_util.verify_signed_payload') + @mock.patch('certbot.ocsp.requests.post') + @mock.patch('certbot.ocsp.ocsp.load_der_ocsp_response') + def test_revoke_resiliency(self, mock_ocsp_response, mock_post, mock_check): + # Server return an invalid HTTP response + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=400) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # OCSP response in invalid + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.UNAUTHORIZED) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # OCSP response is valid, but certificate status is unknown + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # The OCSP response says that the certificate is revoked, but certificate + # does not contain the OCSP extension. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + with mock.patch('cryptography.x509.Extensions.get_extension_for_class', + side_effect=x509.ExtensionNotFound( + 'Not found', x509.AuthorityInformationAccessOID.OCSP)): + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # Valid response, OCSP extension is present, + # but OCSP response uses an unsupported signature. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = UnsupportedAlgorithm('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # And now, the signature itself is invalid. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = InvalidSignature('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # Finally, assertion error on OCSP response validity + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = AssertionError('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + +def _construct_mock_ocsp_response(certificate_status, response_status): + cert = x509.load_pem_x509_certificate( + test_util.load_vector('google_certificate.pem'), default_backend()) + issuer = x509.load_pem_x509_certificate( + test_util.load_vector('google_issuer_certificate.pem'), default_backend()) + builder = ocsp_lib.OCSPRequestBuilder() + builder = builder.add_certificate(cert, issuer, hashes.SHA1()) + request = builder.build() + + return mock.Mock( + response_status=response_status, + certificate_status=certificate_status, + serial_number=request.serial_number, + issuer_key_hash=request.issuer_key_hash, + issuer_name_hash=request.issuer_name_hash, + hash_algorithm=hashes.SHA1(), + next_update=datetime.now() + timedelta(days=1), + this_update=datetime.now() - timedelta(days=1), + signature_algorithm_oid=x509.oid.SignatureAlgorithmOID.RSA_WITH_SHA1, + ) + + # pylint: disable=line-too-long openssl_confused = ("", """ /etc/letsencrypt/live/example.org/cert.pem: good @@ -165,5 +294,6 @@ revoked """, """Response verify OK""") + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/testdata/google_certificate.pem b/certbot/tests/testdata/google_certificate.pem new file mode 100644 index 000000000..c26fea0b1 --- /dev/null +++ b/certbot/tests/testdata/google_certificate.pem @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHQjCCBiqgAwIBAgIQCgYwQn9bvO1pVzllk7ZFHzANBgkqhkiG9w0BAQsFADB1 +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMTQwMgYDVQQDEytEaWdpQ2VydCBTSEEyIEV4dGVuZGVk +IFZhbGlkYXRpb24gU2VydmVyIENBMB4XDTE4MDUwODAwMDAwMFoXDTIwMDYwMzEy +MDAwMFowgccxHTAbBgNVBA8MFFByaXZhdGUgT3JnYW5pemF0aW9uMRMwEQYLKwYB +BAGCNzwCAQMTAlVTMRkwFwYLKwYBBAGCNzwCAQITCERlbGF3YXJlMRAwDgYDVQQF +Ewc1MTU3NTUwMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQG +A1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMuMRMwEQYD +VQQDEwpnaXRodWIuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +xjyq8jyXDDrBTyitcnB90865tWBzpHSbindG/XqYQkzFMBlXmqkzC+FdTRBYyneZ +w5Pz+XWQvL+74JW6LsWNc2EF0xCEqLOJuC9zjPAqbr7uroNLghGxYf13YdqbG5oj +/4x+ogEG3dF/U5YIwVr658DKyESMV6eoYV9mDVfTuJastkqcwero+5ZAKfYVMLUE +sMwFtoTDJFmVf6JlkOWwsxp1WcQ/MRQK1cyqOoUFUgYylgdh3yeCDPeF22Ax8AlQ +xbcaI+GwfQL1FB7Jy+h+KjME9lE/UpgV6Qt2R1xNSmvFCBWu+NFX6epwFP/JRbkM +fLz0beYFUvmMgLtwVpEPSwIDAQABo4IDeTCCA3UwHwYDVR0jBBgwFoAUPdNQpdag +re7zSmAKZdMh1Pj41g8wHQYDVR0OBBYEFMnCU2FmnV+rJfQmzQ84mqhJ6kipMCUG +A1UdEQQeMByCCmdpdGh1Yi5jb22CDnd3dy5naXRodWIuY29tMA4GA1UdDwEB/wQE +AwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0fBG4wbDA0 +oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItZXYtc2VydmVyLWcy +LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTItZXYtc2Vy +dmVyLWcyLmNybDBLBgNVHSAERDBCMDcGCWCGSAGG/WwCATAqMCgGCCsGAQUFBwIB +FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAcGBWeBDAEBMIGIBggrBgEF +BQcBAQR8MHowJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBS +BggrBgEFBQcwAoZGaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0 +U0hBMkV4dGVuZGVkVmFsaWRhdGlvblNlcnZlckNBLmNydDAMBgNVHRMBAf8EAjAA +MIIBfgYKKwYBBAHWeQIEAgSCAW4EggFqAWgAdgCkuQmQtBhYFIe7E6LMZ3AKPDWY +BPkb37jjd80OyA3cEAAAAWNBYm0KAAAEAwBHMEUCIQDRZp38cTWsWH2GdBpe/uPT +Wnsu/m4BEC2+dIcvSykZYgIgCP5gGv6yzaazxBK2NwGdmmyuEFNSg2pARbMJlUFg +U5UAdgBWFAaaL9fC7NP14b1Esj7HRna5vJkRXMDvlJhV1onQ3QAAAWNBYm0tAAAE +AwBHMEUCIQCi7omUvYLm0b2LobtEeRAYnlIo7n6JxbYdrtYdmPUWJQIgVgw1AZ51 +vK9ENinBg22FPxb82TvNDO05T17hxXRC2IYAdgC72d+8H4pxtZOUI5eqkntHOFeV +CqtS6BqQlmQ2jh7RhQAAAWNBYm3fAAAEAwBHMEUCIQChzdTKUU2N+XcqcK0OJYrN +8EYynloVxho4yPk6Dq3EPgIgdNH5u8rC3UcslQV4B9o0a0w204omDREGKTVuEpxG +eOQwDQYJKoZIhvcNAQELBQADggEBAHAPWpanWOW/ip2oJ5grAH8mqQfaunuCVE+v +ac+88lkDK/LVdFgl2B6kIHZiYClzKtfczG93hWvKbST4NRNHP9LiaQqdNC17e5vN +HnXVUGw+yxyjMLGqkgepOnZ2Rb14kcTOGp4i5AuJuuaMwXmCo7jUwPwfLe1NUlVB +Kqg6LK0Hcq4K0sZnxE8HFxiZ92WpV2AVWjRMEc/2z2shNoDvxvFUYyY1Oe67xINk +myQKc+ygSBZzyLnXSFVWmHr3u5dcaaQGGAR42v6Ydr4iL38Hd4dOiBma+FXsXBIq +WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik= +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/google_issuer_certificate.pem b/certbot/tests/testdata/google_issuer_certificate.pem new file mode 100644 index 000000000..50db47bc4 --- /dev/null +++ b/certbot/tests/testdata/google_issuer_certificate.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEXDCCA0SgAwIBAgINAeOpMBz8cgY4P5pTHTANBgkqhkiG9w0BAQsFADBMMSAw +HgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEGA1UEChMKR2xvYmFs +U2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjAeFw0xNzA2MTUwMDAwNDJaFw0yMTEy +MTUwMDAwNDJaMFQxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVHb29nbGUgVHJ1c3Qg +U2VydmljZXMxJTAjBgNVBAMTHEdvb2dsZSBJbnRlcm5ldCBBdXRob3JpdHkgRzMw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKUkvqHv/OJGuo2nIYaNVW +XQ5IWi01CXZaz6TIHLGp/lOJ+600/4hbn7vn6AAB3DVzdQOts7G5pH0rJnnOFUAK +71G4nzKMfHCGUksW/mona+Y2emJQ2N+aicwJKetPKRSIgAuPOB6Aahh8Hb2XO3h9 +RUk2T0HNouB2VzxoMXlkyW7XUR5mw6JkLHnA52XDVoRTWkNty5oCINLvGmnRsJ1z +ouAqYGVQMc/7sy+/EYhALrVJEA8KbtyX+r8snwU5C1hUrwaW6MWOARa8qBpNQcWT +kaIeoYvy/sGIJEmjR0vFEwHdp1cSaWIr6/4g72n7OqXwfinu7ZYW97EfoOSQJeAz +AgMBAAGjggEzMIIBLzAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUH +AwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFHfCuFCa +Z3Z2sS3ChtCDoH6mfrpLMB8GA1UdIwQYMBaAFJviB1dnHB7AagbeWbSaLd/cGYYu +MDUGCCsGAQUFBwEBBCkwJzAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AucGtpLmdv +b2cvZ3NyMjAyBgNVHR8EKzApMCegJaAjhiFodHRwOi8vY3JsLnBraS5nb29nL2dz +cjIvZ3NyMi5jcmwwPwYDVR0gBDgwNjA0BgZngQwBAgIwKjAoBggrBgEFBQcCARYc +aHR0cHM6Ly9wa2kuZ29vZy9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEA +HLeJluRT7bvs26gyAZ8so81trUISd7O45skDUmAge1cnxhG1P2cNmSxbWsoiCt2e +ux9LSD+PAj2LIYRFHW31/6xoic1k4tbWXkDCjir37xTTNqRAMPUyFRWSdvt+nlPq +wnb8Oa2I/maSJukcxDjNSfpDh/Bd1lZNgdd/8cLdsE3+wypufJ9uXO1iQpnh9zbu +FIwsIONGl1p3A8CgxkqI/UAih3JaGOqcpcdaCIzkBaR9uYQ1X4k2Vg5APRLouzVy +7a8IVk6wuy6pm+T7HT4LY8ibS5FEZlfAFLSW8NwsVz9SBK2Vqn1N0PIMn5xA6NZV +c7o835DLAFshEWfC7TIe3g== +-----END CERTIFICATE----- diff --git a/certbot/util.py b/certbot/util.py index 416075ce8..097593e9d 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -62,7 +62,7 @@ def run_script(params, log=logger.error): """Run the script with the given params. :param list params: List of parameters to pass to Popen - :param logging.Logger log: Logger to use for errors + :param callable log: Logger method to use for errors """ try: diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 02656521f..19aaa4eeb 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1034,26 +1034,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 1fac78836..dff57dfd5 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -60,26 +60,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 630571148..9d1ae9ec2 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -15,9 +15,11 @@ command -v python > /dev/null || (echo "Error, python executable is not in the P . ./tests/integration/_common.sh export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx +CURRENT_DIR="$(pwd)" cleanup_and_exit() { EXIT_STATUS=$? + cd $CURRENT_DIR if SERVER_STILL_RUNNING=`ps -p $python_server_pid -o pid=` then echo Kill server subprocess, left running by abnormal exit @@ -527,3 +529,56 @@ if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then fi coverage report --fail-under 64 --include 'certbot/*' --show-missing + +# Test OCSP status + +## OCSP 1: Check stale OCSP status +pushd ./tests/integration + +OUT=`common certificates --config-dir sample-config` +TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l` +EXPIRED=`echo "$OUT" | grep EXPIRED | wc -l` + +if [ "$TEST_CERTS" != 2 ] ; then + echo "Did not find two test certs as expected ($TEST_CERTS)" + exit 1 +fi + +if [ "$EXPIRED" != 2 ] ; then + echo "Did not find two test certs as expected ($EXPIRED)" + exit 1 +fi + +popd + +## OSCP 2: Check live certificate OCSP status (VALID) +common --domains le-ocsp-check.wtf +OUT=`common certificates` +VALID=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep VALID | wc -l` +EXPIRED=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep EXPIRED | wc -l` + +if [ "$VALID" != 1 ] ; then + echo "Expected le-ocsp-check.wtf to be VALID" + exit 1 +fi + +if [ "$EXPIRED" != 0 ] ; then + echo "Did not expect le-ocsp-check.wtf to be EXPIRED" + exit 1 +fi + +## OSCP 3: Check live certificate OCSP status (REVOKED) +common revoke --cert-name le-ocsp-check.wtf --no-delete-after-revoke +OUT=`common certificates` +INVALID=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep INVALID | wc -l` +REVOKED=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep REVOKED | wc -l` + +if [ "$INVALID" != 1 ] ; then + echo "Expected le-ocsp-check.wtf to be INVALID" + exit 1 +fi + +if [ "$REVOKED" != 1 ] ; then + echo "Expected le-ocsp-check.wtf to be REVOKED" + exit 1 +fi diff --git a/tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json b/tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json similarity index 100% rename from tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json rename to tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json diff --git a/tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json b/tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json similarity index 100% rename from tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json rename to tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json diff --git a/tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json b/tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json similarity index 100% rename from tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json rename to tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/cert1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/cert1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/cert1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/cert1.pem diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/chain1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/chain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/chain1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/chain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/fullchain1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/fullchain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/fullchain1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/fullchain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/privkey1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/privkey1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/privkey1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/privkey1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/cert1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/cert1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/cert1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/cert1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/chain1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/chain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/chain1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/chain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/fullchain1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/fullchain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/fullchain1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/fullchain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/privkey1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/privkey1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/privkey1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/privkey1.pem diff --git a/tests/letstest/testdata/sample-config/csr/0000_csr-certbot.pem b/tests/integration/sample-config/csr/0000_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0000_csr-certbot.pem rename to tests/integration/sample-config/csr/0000_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/csr/0001_csr-certbot.pem b/tests/integration/sample-config/csr/0001_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0001_csr-certbot.pem rename to tests/integration/sample-config/csr/0001_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/csr/0002_csr-certbot.pem b/tests/integration/sample-config/csr/0002_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0002_csr-certbot.pem rename to tests/integration/sample-config/csr/0002_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/csr/0003_csr-certbot.pem b/tests/integration/sample-config/csr/0003_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0003_csr-certbot.pem rename to tests/integration/sample-config/csr/0003_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0000_key-certbot.pem b/tests/integration/sample-config/keys/0000_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0000_key-certbot.pem rename to tests/integration/sample-config/keys/0000_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0001_key-certbot.pem b/tests/integration/sample-config/keys/0001_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0001_key-certbot.pem rename to tests/integration/sample-config/keys/0001_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0002_key-certbot.pem b/tests/integration/sample-config/keys/0002_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0002_key-certbot.pem rename to tests/integration/sample-config/keys/0002_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0003_key-certbot.pem b/tests/integration/sample-config/keys/0003_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0003_key-certbot.pem rename to tests/integration/sample-config/keys/0003_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/README b/tests/integration/sample-config/live/a.encryption-example.com/README similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/README rename to tests/integration/sample-config/live/a.encryption-example.com/README diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/cert.pem b/tests/integration/sample-config/live/a.encryption-example.com/cert.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/cert.pem rename to tests/integration/sample-config/live/a.encryption-example.com/cert.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/chain.pem b/tests/integration/sample-config/live/a.encryption-example.com/chain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/chain.pem rename to tests/integration/sample-config/live/a.encryption-example.com/chain.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/fullchain.pem b/tests/integration/sample-config/live/a.encryption-example.com/fullchain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/fullchain.pem rename to tests/integration/sample-config/live/a.encryption-example.com/fullchain.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/privkey.pem b/tests/integration/sample-config/live/a.encryption-example.com/privkey.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/privkey.pem rename to tests/integration/sample-config/live/a.encryption-example.com/privkey.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/README b/tests/integration/sample-config/live/b.encryption-example.com/README similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/README rename to tests/integration/sample-config/live/b.encryption-example.com/README diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/cert.pem b/tests/integration/sample-config/live/b.encryption-example.com/cert.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/cert.pem rename to tests/integration/sample-config/live/b.encryption-example.com/cert.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/chain.pem b/tests/integration/sample-config/live/b.encryption-example.com/chain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/chain.pem rename to tests/integration/sample-config/live/b.encryption-example.com/chain.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/fullchain.pem b/tests/integration/sample-config/live/b.encryption-example.com/fullchain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/fullchain.pem rename to tests/integration/sample-config/live/b.encryption-example.com/fullchain.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/privkey.pem b/tests/integration/sample-config/live/b.encryption-example.com/privkey.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/privkey.pem rename to tests/integration/sample-config/live/b.encryption-example.com/privkey.pem diff --git a/tests/letstest/testdata/sample-config/options-ssl-apache.conf b/tests/integration/sample-config/options-ssl-apache.conf similarity index 100% rename from tests/letstest/testdata/sample-config/options-ssl-apache.conf rename to tests/integration/sample-config/options-ssl-apache.conf diff --git a/tests/letstest/testdata/sample-config/renewal/a.encryption-example.com.conf b/tests/integration/sample-config/renewal/a.encryption-example.com.conf similarity index 100% rename from tests/letstest/testdata/sample-config/renewal/a.encryption-example.com.conf rename to tests/integration/sample-config/renewal/a.encryption-example.com.conf diff --git a/tests/letstest/testdata/sample-config/renewal/b.encryption-example.com.conf b/tests/integration/sample-config/renewal/b.encryption-example.com.conf similarity index 100% rename from tests/letstest/testdata/sample-config/renewal/b.encryption-example.com.conf rename to tests/integration/sample-config/renewal/b.encryption-example.com.conf diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh index 2cbe66a83..081ff3829 100755 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh @@ -17,27 +17,6 @@ letsencrypt-auto certonly --no-self-upgrade -v --standalone --debug \ --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL -# we have to jump through some hoops to cope with relative paths in renewal -# conf files ... -# 1. be in the right directory -cd tests/letstest/testdata/ - -# 2. refer to the config with the same level of relativity that it itself -# contains :/ -OUT=`letsencrypt-auto certificates --config-dir sample-config -v --no-self-upgrade` -TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l` -REVOKED=`echo "$OUT" | grep REVOKED | wc -l` - -if [ "$TEST_CERTS" != 2 ] ; then - echo "Did not find two test certs as expected ($TEST_CERTS)" - exit 1 -fi - -if [ "$REVOKED" != 1 ] ; then - echo "Did not find one revoked cert as expected ($REVOKED)" - exit 1 -fi - if ! letsencrypt-auto --help --no-self-upgrade | grep -F "letsencrypt-auto [SUBCOMMAND]"; then echo "letsencrypt-auto not included in help output!" exit 1 From a809c3697d433515517ba07b85c5ba42b34ff7c1 Mon Sep 17 00:00:00 2001 From: schoen Date: Wed, 27 Feb 2019 16:32:57 -0800 Subject: [PATCH 18/44] Warn sysadmins about privilege escalation risk (#6795) --- docs/install.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/install.rst b/docs/install.rst index 35b262482..eae40c1f0 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -11,6 +11,8 @@ About Certbot *Certbot is meant to be run directly on a web server*, normally by a system administrator. In most cases, running Certbot on your personal computer is not a useful option. The instructions below relate to installing and running Certbot on a server. +System administrators can use Certbot directly to request certificates; they should *not* allow unprivileged users to run arbitrary Certbot commands as ``root``, because Certbot allows its user to specify arbitrary file locations and run arbitrary scripts. + Certbot is packaged for many common operating systems and web servers. Check whether ``certbot`` (or ``letsencrypt``) is packaged for your web server's OS by visiting certbot.eff.org_, where you will also find the correct installation instructions for From 5e849e03f6e8dd2aca31888f9482da63359bb961 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 28 Feb 2019 20:35:58 +0100 Subject: [PATCH 19/44] [Windows] Fix pipstrap (#6775) * Fix pipstrap on windows * Pipstrap pin setuptools version, so explicit install it is not needed anymore. * Rebuild letsencrypt-auto source * Use sys.executable in pipstrap to allow straightforward execution in a venv by choosing the python interpreter from this venv. * Update letsencrypt-auto * Simulate test-everything * Revert "Simulate test-everything" This reverts commit b62c4d719a6e741cb11126c7490097a79c68cf4d. * Clean pipstrap code --- letsencrypt-auto-source/letsencrypt-auto | 22 +++++++++--------- letsencrypt-auto-source/pieces/pipstrap.py | 22 +++++++++--------- tools/_venv_common.py | 27 ++++++++-------------- 3 files changed, 32 insertions(+), 39 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 19aaa4eeb..c390f28ec 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1221,7 +1221,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -1241,7 +1240,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1263,7 +1262,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -1344,7 +1343,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -1354,12 +1354,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -1372,7 +1372,7 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 6a00dd9cb..f04ebe6fe 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -23,7 +23,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -43,7 +42,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -65,7 +64,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -146,7 +145,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -156,12 +156,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -174,4 +174,4 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 540842773..09383b4c0 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -107,13 +107,13 @@ def subprocess_with_print(cmd, env=os.environ, shell=False): subprocess.check_call(cmd, env=env, shell=shell) -def get_venv_bin_path(venv_path): +def get_venv_python_path(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): - return os.path.abspath(os.path.dirname(python_linux)) + return os.path.abspath(python_linux) python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_windows): - return os.path.abspath(os.path.dirname(python_windows)) + return os.path.abspath(python_windows) raise ValueError(( 'Error, could not find python executable in venv path {0}: is it a valid venv ?' @@ -149,17 +149,12 @@ def main(venv_name, venv_args, args): command.extend(shlex.split(venv_args)) subprocess_with_print(command) - # We execute the following commands in the context of the virtual environment, to install - # the packages in it. To do so, we append the venv binary to the PATH that will be used for - # these commands. With this trick, correct python executable will be selected. - new_environ = os.environ.copy() - new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) - subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), - env=new_environ, shell=True) - subprocess_with_print('python -m pip install --upgrade "setuptools>=30.3"', - env=new_environ, shell=True) - subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), - env=new_environ, shell=True) + # Using the python executable from venv, we ensure to execute following commands in this venv. + py_venv = get_venv_python_path(venv_name) + subprocess_with_print([py_venv, os.path.abspath('letsencrypt-auto-source/pieces/pipstrap.py')]) + command = [py_venv, os.path.abspath('tools/pip_install.py')] + command.extend(args) + subprocess_with_print(command) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -179,6 +174,4 @@ def main(venv_name, venv_args, args): if __name__ == '__main__': - main('venv', - '', - sys.argv[1:]) + main('venv', '', sys.argv[1:]) From efc8d49806b14a31d88cfc0f1b6daca1dd373d8d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 28 Feb 2019 21:49:36 +0100 Subject: [PATCH 20/44] Disable rerun feature of Travis (#6800) The rerun capability in a test campaign can be a nice feature. It can also a bad design. It is right that with high level tests, like performance or end-to-end tests, a given test runtime can depend on an external component, outside the scope of the developers, that would spuriously fail. In theses situations, having the capacity to rerun several time a test can be a great benefit. Indeed, as these tests are inherently flaky, the rerun greatly reduces their failure because of external reasons, reducing also the Pierre et le Loup effect (from a well-known french book for children): because tests constantly fail for no reason, you stop to listen to them and to not see when they fail for a real reason. However this not apply to unit tests, or operations about code quality. For theses executions, the flakiness should not exist: a unit test is supposed to have no external dependency. A rerun approach would just hide a situation that is not desirable for this kind of tests Also effectively I see that our tests are usually not flaky: so the only effect of the rerun is to give the failure state about a test three times slower. If a test becomes flaky, this should be fixed, and so be visible immediately in our CI. For these reasons, I remove the travis_retry in the script section of .travis-ci.yml, to call directly tox and let the pipeline fails on the first error. * Disable rerun feature of Travis * Update .travis.yml * Remove completely the retry logic --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0d59ec9e1..4adab1e4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -197,10 +197,10 @@ addons: - nginx-light - openssl -install: "travis_retry $(command -v pip || command -v pip3) install codecov tox" +install: "$(command -v pip || command -v pip3) install codecov tox" script: - - travis_retry tox - - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' + - tox + - '[ -z "${BOULDER_INTEGRATION+x}" ] || (tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' after_success: '[ "$TOXENV" == "py27-cover" ] && codecov' From 841f8efd0aa7dc2ba249b303e6be664d0a6647e3 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 1 Mar 2019 22:18:06 +0100 Subject: [PATCH 21/44] [Unix] Create a framework for certbot integration tests: PART 1 (#6578) * First part * Several optimizations about the docker env setup * Documentation * Various corrections and documentation. Add acme and certbot explicitly as dependencies of certbot-ci. * Correct a variable misinterpreted as a pytest hook * Correct strict parsing option on pebble * Refactor acme setup to be executed from pytest hooks. * Pass TRAVIS env variable to trigger specific xdist logic * Retrigger build. * Work in progress * Config operational * Propagate to xdist * Corrections on acme and misc * Correct subnet for pebble * Remove gobetween, as tls-sni challenges are not tested anymore. * Improve pebble setup. Reduce LOC. * Update acme.py * Optimize acme ca setup, with less temporary assets * Silent setup * Clean code * Remove unused workspace * Use default network driver * Remove bridge * Update package documentation * Remove rerun capability for integration tests, not needed. * Add documentation * Variable for all ports and subnets used by the stack * Update certbot-ci/certbot_integration_tests/conftest.py Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/utils/acme.py Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/utils/misc.py Co-Authored-By: adferrand * Update tox.ini Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/utils/misc.py Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/utils/acme.py Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/utils/acme.py Co-Authored-By: adferrand * Update certbot-ci/certbot_integration_tests/conftest.py Co-Authored-By: adferrand * Rename to acme_server * Add comment * Refactor in a unique context fixture * Remove the need of CERTBOT_ACME_XDIST environment variable * Remove nonstrict/strict options in pebble * Clean dependencies * Clean tox * Change function name * Add comment about coveragerc specificities * Change a comment. * Update setup.py * Update conftest.py * Use the production-ready docker-compose.yml file for Pebble * New style class * Tune pebble to have a stable test environment * Pin a dependency --- .../certbot_integration_tests/.coveragerc | 8 + .../certbot_integration_tests/__init__.py | 1 + .../certbot_tests/__init__.py | 0 .../certbot_tests/context.py | 16 ++ .../certbot_tests/test_main.py | 40 ++++ .../certbot_integration_tests/conftest.py | 92 +++++++++ .../nginx_tests/__init__.py | 0 .../nginx_tests/context.py | 5 + .../nginx_tests/test_main.py | 17 ++ .../utils/__init__.py | 0 .../utils/acme_server.py | 194 ++++++++++++++++++ .../certbot_integration_tests/utils/misc.py | 45 ++++ certbot-ci/setup.py | 45 ++++ tools/dev_constraints.txt | 1 + tox.ini | 12 ++ 15 files changed, 476 insertions(+) create mode 100644 certbot-ci/certbot_integration_tests/.coveragerc create mode 100644 certbot-ci/certbot_integration_tests/__init__.py create mode 100644 certbot-ci/certbot_integration_tests/certbot_tests/__init__.py create mode 100644 certbot-ci/certbot_integration_tests/certbot_tests/context.py create mode 100644 certbot-ci/certbot_integration_tests/certbot_tests/test_main.py create mode 100644 certbot-ci/certbot_integration_tests/conftest.py create mode 100644 certbot-ci/certbot_integration_tests/nginx_tests/__init__.py create mode 100644 certbot-ci/certbot_integration_tests/nginx_tests/context.py create mode 100644 certbot-ci/certbot_integration_tests/nginx_tests/test_main.py create mode 100644 certbot-ci/certbot_integration_tests/utils/__init__.py create mode 100644 certbot-ci/certbot_integration_tests/utils/acme_server.py create mode 100644 certbot-ci/certbot_integration_tests/utils/misc.py create mode 100644 certbot-ci/setup.py diff --git a/certbot-ci/certbot_integration_tests/.coveragerc b/certbot-ci/certbot_integration_tests/.coveragerc new file mode 100644 index 000000000..de36d4e02 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/.coveragerc @@ -0,0 +1,8 @@ +[run] +# Avoid false warnings because certbot packages are not installed in the thread that executes +# the coverage: indeed, certbot is launched as a CLI from a subprocess. +disable_warnings = module-not-imported,no-data-collected + +[report] +# Exclude unit tests in coverage during integration tests. +omit = **/*_test.py,**/tests/*,**/certbot_nginx/parser_obj.py diff --git a/certbot-ci/certbot_integration_tests/__init__.py b/certbot-ci/certbot_integration_tests/__init__.py new file mode 100644 index 000000000..434a85a23 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/__init__.py @@ -0,0 +1 @@ +"""Package certbot_integration_test is for tests that require a live acme ca server instance""" diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py b/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py new file mode 100644 index 000000000..9045cd37d --- /dev/null +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -0,0 +1,16 @@ +class IntegrationTestsContext(object): + """General fixture describing a certbot integration tests context""" + def __init__(self, request): + self.request = request + if hasattr(request.config, 'slaveinput'): # Worker node + self.worker_id = request.config.slaveinput['slaveid'] + self.acme_xdist = request.config.slaveinput['acme_xdist'] + else: # Primary node + self.worker_id = 'primary' + self.acme_xdist = request.config.acme_xdist + self.directory_url = self.acme_xdist['directory_url'] + self.tls_alpn_01_port = self.acme_xdist['https_port'][self.worker_id] + self.http_01_port = self.acme_xdist['http_port'][self.worker_id] + + def cleanup(self): + pass diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py new file mode 100644 index 000000000..5b0981b36 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -0,0 +1,40 @@ +import requests +import urllib3 + +import pytest + +from certbot_integration_tests.certbot_tests import context as certbot_context + + +@pytest.fixture() +def context(request): + # Fixture request is a built-in pytest fixture describing current test request. + integration_test_context = certbot_context.IntegrationTestsContext(request) + try: + yield integration_test_context + finally: + integration_test_context.cleanup() + + +def test_hello_1(context): + assert context.http_01_port + assert context.tls_alpn_01_port + try: + response = requests.get(context.directory_url, verify=False) + response.raise_for_status() + assert response.json() + response.close() + except urllib3.exceptions.InsecureRequestWarning: + pass + + +def test_hello_2(context): + assert context.http_01_port + assert context.tls_alpn_01_port + try: + response = requests.get(context.directory_url, verify=False) + response.raise_for_status() + assert response.json() + response.close() + except urllib3.exceptions.InsecureRequestWarning: + pass diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py new file mode 100644 index 000000000..892c16266 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -0,0 +1,92 @@ +""" +General conftest for pytest execution of all integration tests lying +in the certbot_integration tests package. +As stated by pytest documentation, conftest module is used to set on +for a directory a specific configuration using built-in pytest hooks. + +See https://docs.pytest.org/en/latest/reference.html#hook-reference +""" +import contextlib +import sys +import subprocess + +from certbot_integration_tests.utils import acme_server as acme_lib + + +def pytest_addoption(parser): + """ + Standard pytest hook to add options to the pytest parser. + :param parser: current pytest parser that will be used on the CLI + """ + parser.addoption('--acme-server', default='pebble', + choices=['boulder-v1', 'boulder-v2', 'pebble'], + help='select the ACME server to use (boulder-v1, boulder-v2, ' + 'pebble), defaulting to pebble') + + +def pytest_configure(config): + """ + Standard pytest hook used to add a configuration logic for each node of a pytest run. + :param config: the current pytest configuration + """ + if not hasattr(config, 'slaveinput'): # If true, this is the primary node + with _print_on_err(): + config.acme_xdist = _setup_primary_node(config) + + +def pytest_configure_node(node): + """ + Standard pytest-xdist hook used to configure a worker node. + :param node: current worker node + """ + node.slaveinput['acme_xdist'] = node.config.acme_xdist + + +@contextlib.contextmanager +def _print_on_err(): + """ + During pytest-xdist setup, stdout is used for nodes communication, so print is useless. + However, stderr is still available. This context manager transfers stdout to stderr + for the duration of the context, allowing to display prints to the user. + """ + old_stdout = sys.stdout + sys.stdout = sys.stderr + try: + yield + finally: + sys.stdout = old_stdout + + +def _setup_primary_node(config): + """ + Setup the environment for integration tests. + Will: + - check runtime compatiblity (Docker, docker-compose, Nginx) + - create a temporary workspace and the persistent GIT repositories space + - configure and start paralleled ACME CA servers using Docker + - transfer ACME CA servers configurations to pytest nodes using env variables + :param config: Configuration of the pytest primary node + """ + # Check for runtime compatibility: some tools are required to be available in PATH + try: + subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + raise ValueError('Error: docker is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.') + + try: + subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.') + + # Parameter numprocesses is added to option by pytest-xdist + workers = ['primary'] if not config.option.numprocesses\ + else ['gw{0}'.format(i) for i in range(config.option.numprocesses)] + + # By calling setup_acme_server we ensure that all necessary acme server instances will be + # fully started. This runtime is reflected by the acme_xdist returned. + acme_xdist = acme_lib.setup_acme_server(config.option.acme_server, workers) + print('ACME xdist config:\n{0}'.format(acme_xdist)) + + return acme_xdist diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/__init__.py b/certbot-ci/certbot_integration_tests/nginx_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/context.py b/certbot-ci/certbot_integration_tests/nginx_tests/context.py new file mode 100644 index 000000000..6d7d6012b --- /dev/null +++ b/certbot-ci/certbot_integration_tests/nginx_tests/context.py @@ -0,0 +1,5 @@ +from certbot_integration_tests.certbot_tests import context as certbot_context + + +class IntegrationTestsContext(certbot_context.IntegrationTestsContext): + """General fixture describing a certbot-nginx integration tests context""" diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py new file mode 100644 index 000000000..472e5e7b7 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -0,0 +1,17 @@ +import pytest + +from certbot_integration_tests.nginx_tests import context as nginx_context + + +@pytest.fixture() +def context(request): + # Fixture request is a built-in pytest fixture describing current test request. + integration_test_context = nginx_context.IntegrationTestsContext(request) + try: + yield integration_test_context + finally: + integration_test_context.cleanup() + + +def test_hello(context): + print(context.directory_url) diff --git a/certbot-ci/certbot_integration_tests/utils/__init__.py b/certbot-ci/certbot_integration_tests/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py new file mode 100644 index 000000000..33ef05194 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -0,0 +1,194 @@ +"""Module to setup an ACME CA server environment able to run multiple tests in parallel""" +from __future__ import print_function +import tempfile +import atexit +import os +import subprocess +import shutil +import sys +from os.path import join + +import requests +import json +import yaml + +from certbot_integration_tests.utils import misc + +# These ports are set implicitly in the docker-compose.yml files of Boulder/Pebble. +CHALLTESTSRV_PORT = 8055 +HTTP_01_PORT = 5002 + + +def setup_acme_server(acme_server, nodes): + """ + This method will setup an ACME CA server and an HTTP reverse proxy instance, to allow parallel + execution of integration tests against the unique http-01 port expected by the ACME CA server. + Instances are properly closed and cleaned when the Python process exits using atexit. + Typically all pytest integration tests will be executed in this context. + This method returns an object describing ports and directory url to use for each pytest node + with the relevant pytest xdist node. + :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) + :param str[] nodes: list of node names that will be setup by pytest xdist + :return: a dict describing the challenge ports that have been setup for the nodes + :rtype: dict + """ + acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' + acme_xdist = _construct_acme_xdist(acme_server, nodes) + workspace = _construct_workspace(acme_type) + + _prepare_traefik_proxy(workspace, acme_xdist) + _prepare_acme_server(workspace, acme_type, acme_xdist) + + return acme_xdist + + +def _construct_acme_xdist(acme_server, nodes): + """Generate and return the acme_xdist dict""" + acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT} + + # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble. + if acme_server == 'pebble': + acme_xdist['directory_url'] = 'https://localhost:14000/dir' + else: # boulder + port = 4001 if acme_server == 'boulder-v2' else 4000 + acme_xdist['directory_url'] = 'http://localhost:{0}/directory'.format(port) + + acme_xdist['http_port'] = {node: port for (node, port) + in zip(nodes, range(5200, 5200 + len(nodes)))} + acme_xdist['https_port'] = {node: port for (node, port) + in zip(nodes, range(5100, 5100 + len(nodes)))} + + return acme_xdist + + +def _construct_workspace(acme_type): + """Create a temporary workspace for integration tests stack""" + workspace = tempfile.mkdtemp() + + def cleanup(): + """Cleanup function to call that will teardown relevant dockers and their configuration.""" + for instance in [acme_type, 'traefik']: + print('=> Tear down the {0} instance...'.format(instance)) + instance_path = join(workspace, instance) + try: + if os.path.isfile(join(instance_path, 'docker-compose.yml')): + _launch_command(['docker-compose', 'down'], cwd=instance_path) + except subprocess.CalledProcessError: + pass + print('=> Finished tear down of {0} instance.'.format(acme_type)) + + shutil.rmtree(workspace) + + # Here with atexit we ensure that clean function is called no matter what. + atexit.register(cleanup) + + return workspace + + +def _prepare_acme_server(workspace, acme_type, acme_xdist): + """Configure and launch the ACME server, Boulder or Pebble""" + print('=> Starting {0} instance deployment...'.format(acme_type)) + instance_path = join(workspace, acme_type) + try: + # Load Boulder/Pebble from git, that includes a docker-compose.yml ready for production. + _launch_command(['git', 'clone', 'https://github.com/letsencrypt/{0}'.format(acme_type), + '--single-branch', '--depth=1', instance_path]) + if acme_type == 'boulder': + # Allow Boulder to ignore usual limit rate policies, useful for tests. + os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'), + join(instance_path, 'test/rate-limit-policies.yml')) + if acme_type == 'pebble': + # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid + # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment. + with open(os.path.join(instance_path, 'docker-compose.yml'), 'r') as file_handler: + config = yaml.load(file_handler.read()) + + config['services']['pebble'].setdefault('environment', [])\ + .extend(['PEBBLE_VA_NOSLEEP=1', 'PEBBLE_WFE_NONCEREJECT=0']) + with open(os.path.join(instance_path, 'docker-compose.yml'), 'w') as file_handler: + file_handler.write(yaml.dump(config)) + + # Launch the ACME CA server. + _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path) + + # Wait for the ACME CA server to be up. + print('=> Waiting for {0} instance to respond...'.format(acme_type)) + misc.check_until_timeout(acme_xdist['directory_url']) + + # Configure challtestsrv to answer any A record request with ip of the docker host. + acme_subnet = '10.77.77' if acme_type == 'boulder' else '10.30.50' + response = requests.post('http://localhost:{0}/set-default-ipv4' + .format(acme_xdist['challtestsrv_port']), + json={'ip': '{0}.1'.format(acme_subnet)}) + response.raise_for_status() + + print('=> Finished {0} instance deployment.'.format(acme_type)) + except BaseException: + print('Error while setting up {0} instance.'.format(acme_type)) + raise + + +def _prepare_traefik_proxy(workspace, acme_xdist): + """Configure and launch Traefik, the HTTP reverse proxy""" + print('=> Starting traefik instance deployment...') + instance_path = join(workspace, 'traefik') + traefik_subnet = '10.33.33' + traefik_api_port = 8056 + try: + os.mkdir(instance_path) + + with open(join(instance_path, 'docker-compose.yml'), 'w') as file_h: + file_h.write('''\ +version: '3' +services: + traefik: + image: traefik + command: --api --rest + ports: + - {http_01_port}:80 + - {traefik_api_port}:8080 + networks: + traefiknet: + ipv4_address: {traefik_subnet}.2 +networks: + traefiknet: + ipam: + config: + - subnet: {traefik_subnet}.0/24 +'''.format(traefik_subnet=traefik_subnet, + traefik_api_port=traefik_api_port, + http_01_port=HTTP_01_PORT)) + + _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path) + + misc.check_until_timeout('http://localhost:{0}/api'.format(traefik_api_port)) + config = { + 'backends': { + node: { + 'servers': {node: {'url': 'http://{0}.1:{1}'.format(traefik_subnet, port)}} + } for node, port in acme_xdist['http_port'].items() + }, + 'frontends': { + node: { + 'backend': node, 'passHostHeader': True, + 'routes': {node: {'rule': 'HostRegexp: {{subdomain:.+}}.{0}.wtf'.format(node)}} + } for node in acme_xdist['http_port'].keys() + } + } + response = requests.put('http://localhost:{0}/api/providers/rest'.format(traefik_api_port), + data=json.dumps(config)) + response.raise_for_status() + + print('=> Finished traefik instance deployment.') + except BaseException: + print('Error while setting up traefik instance.') + raise + + +def _launch_command(command, cwd=os.getcwd()): + """Launch silently an OS command, output will be displayed in case of failure""" + try: + subprocess.check_output(command, stderr=subprocess.STDOUT, cwd=cwd, universal_newlines=True) + except subprocess.CalledProcessError as e: + sys.stderr.write(e.output) + raise diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py new file mode 100644 index 000000000..a3b134788 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -0,0 +1,45 @@ +""" +Misc module contains stateless functions that could be used during pytest execution, +or outside during setup/teardown of the integration tests environment. +""" +import os +import time +import contextlib + +import requests + + +def check_until_timeout(url): + """ + Wait and block until given url responds with status 200, or raise an exception + after 150 attempts. + :param str url: the URL to test + :raise ValueError: exception raised after 150 unsuccessful attempts to reach the URL + """ + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + for _ in range(0, 150): + time.sleep(1) + try: + if requests.get(url, verify=False).status_code == 200: + return + except requests.exceptions.ConnectionError: + pass + + raise ValueError('Error, url did not respond after 150 attempts: {0}'.format(url)) + + +@contextlib.contextmanager +def execute_in_given_cwd(cwd): + """ + Context manager that will execute any command in the given cwd after entering context, + and restore current cwd when context is destroyed. + :param str cwd: the path to use as the temporary current workspace for python execution + """ + current_cwd = os.getcwd() + try: + os.chdir(cwd) + yield + finally: + os.chdir(current_cwd) diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py new file mode 100644 index 000000000..595bba69e --- /dev/null +++ b/certbot-ci/setup.py @@ -0,0 +1,45 @@ +from setuptools import setup +from setuptools import find_packages + + +version = '0.32.0.dev0' + +install_requires = [ + 'pytest', + 'pytest-cov', + 'pytest-xdist', + 'pytest-sugar', + 'coverage', + 'requests', + 'pyyaml', +] + +setup( + name='certbot-ci', + version=version, + description="Certbot continuous integration framework", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, +) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 88340cb00..f8c3f4461 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -51,6 +51,7 @@ pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.22.5 +pytest-sugar==0.9.2 python-dateutil==2.6.1 python-digitalocean==1.11 PyYAML==3.13 diff --git a/tox.ini b/tox.ini index b386ebc86..2c5fe0644 100644 --- a/tox.ini +++ b/tox.ini @@ -250,3 +250,15 @@ commands = whitelist_externals = docker-compose passenv = DOCKER_* + +[testenv:integration] +commands = + {[base]pip_install} acme . certbot-nginx certbot-ci + pytest {toxinidir}/certbot-ci/certbot_integration_tests \ + --acme-server={env:ACME_SERVER:pebble} \ + --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \ + --cov-config={toxinidir}/certbot-ci/certbot_integration_tests/.coveragerc \ + -W 'ignore:Unverified HTTPS request' + coverage report --fail-under=65 --show-missing +passenv = + DOCKER_* From 7161e792e80d4b0ffa404199248334b2b34c0235 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 1 Mar 2019 22:54:09 +0100 Subject: [PATCH 22/44] Fix the Nginx configuration during integration tests (#6801) If you execute `tests/lock_test.py` or `tox -e integration` on a fairly recent machine, you will get the following error during tests executing against a live Nginx instance: ``` no "ssl_certificate" is defined in server listening on SSL port while SSL handshaking, client: x.x.x.x, server: y:y:y:y:z ``` Indeed, having no defined ssl certificate for a ssl port would inevitably lead to an error during the handshake SSL process between a client and this mis-configured nginx instance. However it was not a problem one year before, because the handshake was not occurring in practice: the test just need to have a nginx started, and then immediately proceed to modify the configuration with a correct SSL setup. And nginx was able to start with a mis-configuration on SSL. But then this fix has been done: https://trac.nginx.org/nginx/ticket/178 Basically with this, validation of the configuration is done during nginx startup, that will refuse to start with invalid configuration on SSL. Consequently, all related tests are failing with a sufficiently up-to-date nginx. For now, it is not seen on Travis because Ubuntu Trusty is used, with an old Nginx. The PR fixes that, by generating on the fly self-signed certificates in the two impacted tests, and pushing the right parameters in the Nginx configuration. * Fix nginx configuration with self-signed certificates generated on the fly * Fix lint/mypy * Fix old cryptography * Unattended openssl * Update lock_test.py --- .../tests/boulder-integration.conf.sh | 29 ++++++--- certbot-nginx/tests/boulder-integration.sh | 6 +- tests/lock_test.py | 62 +++++++++++++++++-- 3 files changed, 81 insertions(+), 16 deletions(-) diff --git a/certbot-nginx/tests/boulder-integration.conf.sh b/certbot-nginx/tests/boulder-integration.conf.sh index 470eab28e..80cca2682 100755 --- a/certbot-nginx/tests/boulder-integration.conf.sh +++ b/certbot-nginx/tests/boulder-integration.conf.sh @@ -3,16 +3,22 @@ # https://www.exratione.com/2014/03/running-nginx-as-a-non-root-user/ # https://github.com/exratione/non-root-nginx/blob/9a77f62e5d5cb9c9026fd62eece76b9514011019/nginx.conf +# USAGE: ./boulder-integration.conf.sh /path/to/root cert.key cert.pem >> nginx.conf + +ROOT=$1 +CERT_KEY_PATH=$2 +CERT_PATH=$3 + cat < $nginx_conf diff --git a/tests/lock_test.py b/tests/lock_test.py index 0266cf029..aaa8ce2d9 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -2,6 +2,7 @@ from __future__ import print_function import atexit +import datetime import functools import logging import os @@ -11,6 +12,13 @@ import subprocess import sys import tempfile +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +# TODO: once mypy has cryptography types bundled, type: ignore can be removed. +# See https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography.hazmat.primitives import serialization, hashes # type: ignore +from cryptography.hazmat.primitives.asymmetric import rsa + from certbot import lock from certbot import util @@ -102,12 +110,11 @@ def set_up_nginx_dir(root_path): repo_root = check_call('git rev-parse --show-toplevel'.split()).strip() conf_script = os.path.join( repo_root, 'certbot-nginx', 'tests', 'boulder-integration.conf.sh') - # boulder-integration.conf.sh uses the root environment variable as - # the Nginx server root when writing paths - os.environ['root'] = root_path + # Prepare self-signed certificates for Nginx + key_path, cert_path = setup_certificate(root_path) + # Generate Nginx configuration with open(os.path.join(root_path, 'nginx.conf'), 'w') as f: - f.write(check_call(['/bin/sh', conf_script])) - del os.environ['root'] + f.write(check_call(['/bin/sh', conf_script, root_path, key_path, cert_path])) def set_up_command(config_dir, logs_dir, work_dir, nginx_dir): @@ -134,6 +141,51 @@ def set_up_command(config_dir, logs_dir, work_dir, nginx_dir): config_dir, logs_dir, work_dir, nginx_dir).split()) +def setup_certificate(workspace): + """Generate a self-signed certificate for nginx. + :param workspace: path of folder where to put the certificate + :return: tuple containing the key path and certificate path + :rtype: `tuple` + """ + # Generate key + # See comment on cryptography import about type: ignore + private_key = rsa.generate_private_key( # type: ignore + public_exponent=65537, + key_size=2048, + backend=default_backend() + ) + subject = issuer = x509.Name([ + x509.NameAttribute(x509.NameOID.COMMON_NAME, u'nginx.wtf') + ]) + certificate = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + private_key.public_key() + ).serial_number( + 1 + ).not_valid_before( + datetime.datetime.utcnow() + ).not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=1) + ).sign(private_key, hashes.SHA256(), default_backend()) + + key_path = os.path.join(workspace, 'cert.key') + with open(key_path, 'wb') as file_handle: + file_handle.write(private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + )) + + cert_path = os.path.join(workspace, 'cert.pem') + with open(cert_path, 'wb') as file_handle: + file_handle.write(certificate.public_bytes(serialization.Encoding.PEM)) + + return key_path, cert_path + + def test_command(command, directories): """Assert Certbot acquires locks in a specific order. From a468a3b255e326dbfa2b7d629eb20fa275631254 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 1 Mar 2019 23:21:07 +0100 Subject: [PATCH 23/44] Disable build for commits pushed on master (#6804) Fixes #6746. Every commit on master is always the result of a merged PR, that has been tested by Travis. So retesting the merge commit on master is superfluous. This PR uses build conditions to avoid to launch a build for a commit push on master. I also added the equivalent logic for AppVeyor. Builds cannot received conditions, so it needs to be done on init using Exit-AppVeyorBuild. This command does not fail the build, it finishes it prematurely with success. * Disable build for commit pushed on master (PR are still tested of course) * Equivalent exclusion code for AppVeyor --- .travis.yml | 5 +++++ appveyor.yml | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4adab1e4e..108f853aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,11 @@ branches: - /^\d+\.\d+\.x$/ - /^test-.*$/ +# Since master can receive only commits from PR that have already been tested, we avoid with the +# the following condition to launch again a pipeline when the merge commit is pushed to master. +# However master still needs to be set in branches section to allow PR for master to be built. +if: NOT (type = push AND branch = master) + matrix: include: # These environments are always executed diff --git a/appveyor.yml b/appveyor.yml index 2b6b82747..5dcc80ba9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,6 +11,14 @@ branches: - /^\d+\.\d+\.x$/ # Version branches like X.X.X - /^test-.*$/ +init: + # Since master can receive only commits from PR that have already been tested, we avoid with the + # the following condition to launch again a pipeline when the merge commit is pushed to master. + # However master still needs to be set in branches section to allow PR for master to be built. + - ps: | + if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master') + { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild } + install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" From c7f8f15e9b6c51d67f28af96c84d35b814ba2f73 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 1 Mar 2019 16:04:26 -0800 Subject: [PATCH 24/44] Fix spurious test_relevant_values* failures (#6799) * Mock set_by_cli in _test_relevant_values_common. * Empty commit * Revert "Mock set_by_cli in _test_relevant_values_common." This reverts commit 9dfec8dfa9124d573362f282b691b66af2860fe6. * mock less * Use plugin_common code instead of reimplementing. * Revert 2nd implementation. * Simplify certbot.storage.relevant_values() tests. In addition to cleaning up the code a bit, it also removes the problems we've seen in these tests with the global state used in cli.py. --- certbot/tests/cli_test.py | 2 + certbot/tests/storage_test.py | 109 +++++++++++++--------------------- 2 files changed, 44 insertions(+), 67 deletions(-) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index e16a1bdcf..f6669e7be 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -340,6 +340,8 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods config_dir_option = 'config_dir' self.assertFalse(cli.option_was_set( config_dir_option, cli.flag_default(config_dir_option))) + self.assertFalse(cli.option_was_set( + 'authenticator', cli.flag_default('authenticator'))) def test_encode_revocation_reason(self): for reason, code in constants.REVOCATION_REASONS.items(): diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index e61ed2aca..872fb4302 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -12,7 +12,6 @@ import pytz import six import certbot -from certbot import cli from certbot import compat from certbot import errors from certbot.storage import ALL_FOUR @@ -37,6 +36,48 @@ def fill_with_sample_data(rc_object): f.write(kind) +class RelevantValuesTest(unittest.TestCase): + """Tests for certbot.storage.relevant_values.""" + + def setUp(self): + self.values = {"server": "example.org"} + + def _call(self, *args, **kwargs): + from certbot.storage import relevant_values + return relevant_values(*args, **kwargs) + + @mock.patch("certbot.cli.option_was_set") + @mock.patch("certbot.plugins.disco.PluginsRegistry.find_all") + def test_namespace(self, mock_find_all, mock_option_was_set): + mock_find_all.return_value = ["certbot-foo:bar"] + mock_option_was_set.return_value = True + + self.values["certbot_foo:bar_baz"] = 42 + self.assertEqual( + self._call(self.values.copy()), self.values) + + @mock.patch("certbot.cli.option_was_set") + def test_option_set(self, mock_option_was_set): + mock_option_was_set.return_value = True + + self.values["allow_subset_of_names"] = True + self.values["authenticator"] = "apache" + self.values["rsa_key_size"] = 1337 + expected_relevant_values = self.values.copy() + self.values["hello"] = "there" + + self.assertEqual(self._call(self.values), expected_relevant_values) + + @mock.patch("certbot.cli.option_was_set") + def test_option_unset(self, mock_option_was_set): + mock_option_was_set.return_value = False + + expected_relevant_values = self.values.copy() + self.values["rsa_key_size"] = 2048 + + self.assertEqual(self._call(self.values), expected_relevant_values) + + class BaseRenewableCertTest(test_util.ConfigTestCase): """Base class for setting up Renewable Cert tests. @@ -563,72 +604,6 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.save_successor(2, b"newcert", b"new_privkey", b"new chain", self.config) self.assertTrue(mock_chown.called) - def _test_relevant_values_common(self, values): - defaults = dict((option, cli.flag_default(option)) - for option in ("authenticator", "installer", - "rsa_key_size", "server",)) - mock_parser = mock.Mock(args=[], verb="plugins", - defaults=defaults) - - # make a copy to ensure values isn't modified - values = values.copy() - values.setdefault("server", defaults["server"]) - expected_server = values["server"] - - from certbot.storage import relevant_values - with mock.patch("certbot.cli.helpful_parser", mock_parser): - rv = relevant_values(values) - self.assertIn("server", rv) - self.assertEqual(rv.pop("server"), expected_server) - return rv - - def test_relevant_values(self): - """Test that relevant_values() can reject an irrelevant value.""" - self.assertEqual( - self._test_relevant_values_common({"hello": "there"}), {}) - - def test_relevant_values_default(self): - """Test that relevant_values() can reject a default value.""" - option = "rsa_key_size" - values = {option: cli.flag_default(option)} - self.assertEqual(self._test_relevant_values_common(values), {}) - - def test_relevant_values_nondefault(self): - """Test that relevant_values() can retain a non-default value.""" - values = {"rsa_key_size": 12} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_bool(self): - values = {"allow_subset_of_names": True} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_str(self): - values = {"authenticator": "apache"} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_plugins_none(self): - self.assertEqual( - self._test_relevant_values_common( - {"authenticator": None, "installer": None}), {}) - - @mock.patch("certbot.cli.set_by_cli") - @mock.patch("certbot.plugins.disco.PluginsRegistry.find_all") - def test_relevant_values_namespace(self, mock_find_all, mock_set_by_cli): - mock_set_by_cli.return_value = True - mock_find_all.return_value = ["certbot-foo:bar"] - values = {"certbot_foo:bar_baz": 42} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_server(self): - self.assertEqual( - # _test_relevant_values_common handles testing the server - # value and removes it - self._test_relevant_values_common({"server": "example.org"}), {}) - @mock.patch("certbot.storage.relevant_values") def test_new_lineage(self, mock_rv): """Test for new_lineage() class method.""" From 3ed3787bd8792d6ed5c58537a66425a503107721 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 2 Mar 2019 02:03:33 +0100 Subject: [PATCH 25/44] Implement Retry-After, and refactor authorization polling (#6766) Fixes #5789 This PR is about allowing Certbot to respect the Retry-After HTTP header that an ACME CA server can return to a client POSTing to a challenge, to instruct him and retry the request later. However, this feature was not easily implementable in the current code of certbot.auth_handler, because the code became really hard to read. In fact, @bmw was thinking that the code was really deceiving, and a lot of supposed functionalities declared in the comments were in fact not implemented or not functional. So I took the time to understand what was going on, and effectively, most of the code is in fact not usable or not used. Then I did a refactoring against the bare ACME spec about what to do to prepare challenges, instruct the ACME CA server to perform them, then polling regularly the authorization resources until they are decided (valid or invalid). And of course this implementation takes care of Retry-After ^^ I added a lot of comments in the new implementation, to explain what is going on for a future developer. The workflow I used is relying on the relationships between authorizations and challenges states as described in section 7.1.6 of the ACME spec draft: https://datatracker.ietf.org/doc/draft-ietf-acme-acme/ * Clean auth_handler a bit, and implement retry-after. * Remove a debug logger * Correct tests * Fix mypy and lint. Setup max retries and default retry after accordingly. * Ease a comparison in tests * Update documentation * Add tests * Adapt windows coverage threshold to the global LOC reduction * Update certbot/auth_handler.py Co-Authored-By: adferrand * Corrections under review * Correction under review * Update certbot/auth_handler.py Co-Authored-By: adferrand * Corrections under review * Update auth_handler_test.py * Reimplementing user readable report for failed authorizations * Fixes two tests * Fix another test + lint + mypy * Update auth_handler.py * Update auth_handler_test.py * Fix tests * Update certbot/auth_handler.py Co-Authored-By: adferrand * Raise directly the exception on polling timout * Improve interface documentation * Move the wait on top of the loop, to be used initially or after a new loop iteration. Do not wait for negative values. * Always display the report about failed authorizations. * Clarify an exception. * Return, instead of break * Use setdefault * Remove useless assertion * Adapt tests * Improve a test about retry after value. * Update certbot/auth_handler.py Co-Authored-By: adferrand * Add a complete test on best_effort * Add entry to the changelog * Gather all failed authzrs to be reported in one unique report in case of best_effort * Build complete warn/report/raise process about failed authzrs --- CHANGELOG.md | 3 + certbot/auth_handler.py | 392 ++++++++++------------------- certbot/interfaces.py | 13 +- certbot/plugins/common.py | 2 +- certbot/tests/auth_handler_test.py | 313 +++++++++-------------- tox.cover.py | 2 +- 6 files changed, 252 insertions(+), 473 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eea1923d..764fc0946 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * The running of manual plugin hooks is now always included in Certbot's log output. * Tests execution for certbot, certbot-apache and certbot-nginx packages now relies on pytest. +* An ACME CA server may return a "Retry-After" HTTP header on authorization polling, as + specified in the ACME protocol, to indicate when the next polling should occur. Certbot now + reads this header if set and respect its value. * The `acme` module avoids sending the `keyAuthorization` field in the JWS payload when responding to a challenge as the field is not included in the current ACME protocol. To ease the migration path for ACME CA servers, diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 3dfaaf26f..517b56351 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -1,29 +1,23 @@ """ACME AuthHandler.""" -import collections import logging import time +import datetime -import six import zope.component from acme import challenges from acme import messages # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import DefaultDict, Dict, List, Set, Collection +from acme.magic_typing import Dict, List # pylint: enable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot import error_handler from certbot import interfaces - logger = logging.getLogger(__name__) -AnnotatedAuthzr = collections.namedtuple("AnnotatedAuthzr", ["authzr", "achalls"]) -"""Stores an authorization resource and its active annotated challenges.""" - - class AuthHandler(object): """ACME Authorization Handler for a client. @@ -47,242 +41,151 @@ class AuthHandler(object): self.account = account self.pref_challs = pref_challs - def handle_authorizations(self, orderr, best_effort=False): - """Retrieve all authorizations for challenges. - - :param acme.messages.OrderResource orderr: must have - authorizations filled in - :param bool best_effort: Whether or not all authorizations are - required (this is useful in renewal) - - :returns: List of authorization resources - :rtype: list - - :raises .AuthorizationError: If unable to retrieve all - authorizations - + def handle_authorizations(self, orderr, best_effort=False, max_retries=30): """ - aauthzrs = [AnnotatedAuthzr(authzr, []) - for authzr in orderr.authorizations] + Retrieve all authorizations, perform all challenges required to validate + these authorizations, then poll and wait for the authorization to be checked. + :param acme.messages.OrderResource orderr: must have authorizations filled in + :param bool best_effort: if True, not all authorizations need to be validated (eg. renew) + :param int max_retries: maximum number of retries to poll authorizations + :returns: list of all validated authorizations + :rtype: List - self._choose_challenges(aauthzrs) - config = zope.component.getUtility(interfaces.IConfig) - notify = zope.component.getUtility(interfaces.IDisplay).notification + :raises .AuthorizationError: If unable to retrieve all authorizations + """ + authzrs = orderr.authorizations[:] + if not authzrs: + raise errors.AuthorizationError('No authorization to handle.') - # While there are still challenges remaining... - while self._has_challenges(aauthzrs): - with error_handler.ExitHandler(self._cleanup_challenges, aauthzrs): - resp = self._solve_challenges(aauthzrs) - logger.info("Waiting for verification...") + # Retrieve challenges that need to be performed to validate authorizations. + achalls = self._choose_challenges(authzrs) + if not achalls: + return authzrs + + # Starting now, challenges will be cleaned at the end no matter what. + with error_handler.ExitHandler(self._cleanup_challenges, achalls): + # To begin, let's ask the authenticator plugin to perform all challenges. + try: + resps = self.auth.perform(achalls) + + # If debug is on, wait for user input before starting the verification process. + logger.info('Waiting for verification...') + config = zope.component.getUtility(interfaces.IConfig) if config.debug_challenges: + notify = zope.component.getUtility(interfaces.IDisplay).notification notify('Challenges loaded. Press continue to submit to CA. ' 'Pass "-v" for more info about challenges.', pause=True) + except errors.AuthorizationError as error: + logger.critical('Failure in setting up challenges.') + logger.info('Attempting to clean up outstanding challenges...') + raise error + # All challenges should have been processed by the authenticator. + assert len(resps) == len(achalls), 'Some challenges have not been performed.' - # Send all Responses - this modifies achalls - self._respond(aauthzrs, resp, best_effort) + # Inform the ACME CA server that challenges are available for validation. + for achall, resp in zip(achalls, resps): + self.acme.answer_challenge(achall.challb, resp) - # Just make sure all decisions are complete. - self.verify_authzr_complete(aauthzrs) + # Wait for authorizations to be checked. + self._poll_authorizations(authzrs, max_retries, best_effort) - # Only return valid authorizations - ret_val = [aauthzr.authzr for aauthzr in aauthzrs - if aauthzr.authzr.body.status == messages.STATUS_VALID] + # Keep validated authorizations only. If there is none, no certificate can be issued. + authzrs_validated = [authzr for authzr in authzrs + if authzr.body.status == messages.STATUS_VALID] + if not authzrs_validated: + raise errors.AuthorizationError('All challenges have failed.') - if not ret_val: - raise errors.AuthorizationError( - "Challenges failed for all domains") + return authzrs_validated - return ret_val + def _poll_authorizations(self, authzrs, max_retries, best_effort): + """ + Poll the ACME CA server, to wait for confirmation that authorizations have their challenges + all verified. The poll may occur several times, until all authorizations are checked + (valid or invalid), or after a maximum of retries. + """ + authzrs_to_check = {index: (authzr, None) + for index, authzr in enumerate(authzrs)} + authzrs_failed_to_report = [] + # Give an initial second to the ACME CA server to check the authorizations + sleep_seconds = 1 + for _ in range(max_retries): + # Wait for appropriate time (from Retry-After, initial wait, or no wait) + if sleep_seconds > 0: + time.sleep(sleep_seconds) + # Poll all updated authorizations. + authzrs_to_check = {index: self.acme.poll(authzr) for index, (authzr, _) + in authzrs_to_check.items()} + # Update the original list of authzr with the updated authzrs from server. + for index, (authzr, _) in authzrs_to_check.items(): + authzrs[index] = authzr - def _choose_challenges(self, aauthzrs): + # Gather failed authorizations + authzrs_failed = [authzr for authzr, _ in authzrs_to_check.values() + if authzr.body.status == messages.STATUS_INVALID] + for authzr_failed in authzrs_failed: + logger.warning('Challenge failed for domain %s', + authzr_failed.body.identifier.value) + # Accumulating all failed authzrs to build a consolidated report + # on them at the end of the polling. + authzrs_failed_to_report.extend(authzrs_failed) + + # Extract out the authorization already checked for next poll iteration. + # Poll may stop here because there is no pending authorizations anymore. + authzrs_to_check = {index: (authzr, resp) for index, (authzr, resp) + in authzrs_to_check.items() + if authzr.body.status == messages.STATUS_PENDING} + if not authzrs_to_check: + # Polling process is finished, we can leave the loop + break + + # Be merciful with the ACME server CA, check the Retry-After header returned, + # and wait this time before polling again in next loop iteration. + # From all the pending authorizations, we take the greatest Retry-After value + # to avoid polling an authorization before its relevant Retry-After value. + retry_after = max(self.acme.retry_after(resp, 3) + for _, resp in authzrs_to_check.values()) + sleep_seconds = (retry_after - datetime.datetime.now()).total_seconds() + + # In case of failed authzrs, create a report to the user. + if authzrs_failed_to_report: + _report_failed_authzrs(authzrs_failed_to_report, self.account.key) + if not best_effort: + # Without best effort, having failed authzrs is critical and fail the process. + raise errors.AuthorizationError('Some challenges have failed.') + + if authzrs_to_check: + # Here authzrs_to_check is still not empty, meaning we exceeded the max polling attempt. + raise errors.AuthorizationError('All authorizations were not finalized by the CA.') + + def _choose_challenges(self, authzrs): """ Retrieve necessary and pending challenges to satisfy server. NB: Necessary and already validated challenges are not retrieved, as they can be reused for a certificate issuance. """ - pending_authzrs = [aauthzr for aauthzr in aauthzrs - if aauthzr.authzr.body.status != messages.STATUS_VALID] + pending_authzrs = [authzr for authzr in authzrs + if authzr.body.status != messages.STATUS_VALID] + achalls = [] # type: List[achallenges.AnnotatedChallenge] if pending_authzrs: logger.info("Performing the following challenges:") - for aauthzr in pending_authzrs: - aauthzr_challenges = aauthzr.authzr.body.challenges + for authzr in pending_authzrs: + authzr_challenges = authzr.body.challenges if self.acme.acme_version == 1: - combinations = aauthzr.authzr.body.combinations + combinations = authzr.body.combinations else: - combinations = tuple((i,) for i in range(len(aauthzr_challenges))) + combinations = tuple((i,) for i in range(len(authzr_challenges))) path = gen_challenge_path( - aauthzr_challenges, - self._get_chall_pref(aauthzr.authzr.body.identifier.value), + authzr_challenges, + self._get_chall_pref(authzr.body.identifier.value), combinations) - aauthzr_achalls = self._challenge_factory( - aauthzr.authzr, path) - aauthzr.achalls.extend(aauthzr_achalls) + achalls.extend(self._challenge_factory(authzr, path)) - for aauthzr in aauthzrs: - for achall in aauthzr.achalls: - if isinstance(achall.chall, challenges.TLSSNI01): - logger.warning("TLS-SNI-01 is deprecated, and will stop working soon.") - return + if any(isinstance(achall.chall, challenges.TLSSNI01) for achall in achalls): + logger.warning("TLS-SNI-01 is deprecated, and will stop working soon.") - def _has_challenges(self, aauthzrs): - """Do we have any challenges to perform?""" - return any(aauthzr.achalls for aauthzr in aauthzrs) - - def _solve_challenges(self, aauthzrs): - """Get Responses for challenges from authenticators.""" - resp = [] # type: Collection[challenges.ChallengeResponse] - all_achalls = self._get_all_achalls(aauthzrs) - try: - if all_achalls: - resp = self.auth.perform(all_achalls) - except errors.AuthorizationError: - logger.critical("Failure in setting up challenges.") - logger.info("Attempting to clean up outstanding challenges...") - raise - - assert len(resp) == len(all_achalls) - - return resp - - def _get_all_achalls(self, aauthzrs): - """Return all active challenges.""" - all_achalls = [] # type: Collection[challenges.ChallengeResponse] - for aauthzr in aauthzrs: - all_achalls.extend(aauthzr.achalls) - return all_achalls - - def _respond(self, aauthzrs, resp, best_effort): - """Send/Receive confirmation of all challenges. - - .. note:: This method also cleans up the auth_handler state. - - """ - # TODO: chall_update is a dirty hack to get around acme-spec #105 - chall_update = dict() \ - # type: Dict[int, List[achallenges.KeyAuthorizationAnnotatedChallenge]] - self._send_responses(aauthzrs, resp, chall_update) - - # Check for updated status... - self._poll_challenges(aauthzrs, chall_update, best_effort) - - def _send_responses(self, aauthzrs, resps, chall_update): - """Send responses and make sure errors are handled. - - :param aauthzrs: authorizations and the selected annotated challenges - to try and perform - :type aauthzrs: `list` of `AnnotatedAuthzr` - :param resps: challenge responses from the authenticator where - each response at index i corresponds to the annotated - challenge at index i in the list returned by - :func:`_get_all_achalls` - :type resps: `collections.abc.Iterable` of - :class:`~acme.challenges.ChallengeResponse` or `False` or - `None` - :param dict chall_update: parameter that is updated to hold - aauthzr index to list of outstanding solved annotated challenges - - """ - active_achalls = [] - resps_iter = iter(resps) - for i, aauthzr in enumerate(aauthzrs): - for achall in aauthzr.achalls: - # This line needs to be outside of the if block below to - # ensure failed challenges are cleaned up correctly - active_achalls.append(achall) - - resp = next(resps_iter) - # Don't send challenges for None and False authenticator responses - if resp: - self.acme.answer_challenge(achall.challb, resp) - # TODO: answer_challenge returns challr, with URI, - # that can be used in _find_updated_challr - # comparisons... - chall_update.setdefault(i, []).append(achall) - - return active_achalls - - def _poll_challenges(self, aauthzrs, chall_update, - best_effort, min_sleep=3, max_rounds=30): - """Wait for all challenge results to be determined.""" - indices_to_check = set(chall_update.keys()) - comp_indices = set() - rounds = 0 - - while indices_to_check and rounds < max_rounds: - # TODO: Use retry-after... - time.sleep(min_sleep) - all_failed_achalls = set() # type: Set[achallenges.KeyAuthorizationAnnotatedChallenge] - for index in indices_to_check: - comp_achalls, failed_achalls = self._handle_check( - aauthzrs, index, chall_update[index]) - - if len(comp_achalls) == len(chall_update[index]): - comp_indices.add(index) - elif not failed_achalls: - for achall, _ in comp_achalls: - chall_update[index].remove(achall) - # We failed some challenges... damage control - else: - if best_effort: - comp_indices.add(index) - logger.warning( - "Challenge failed for domain %s", - aauthzrs[index].authzr.body.identifier.value) - else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) - - if all_failed_achalls: - _report_failed_challs(all_failed_achalls) - raise errors.FailedChallenges(all_failed_achalls) - - indices_to_check -= comp_indices - comp_indices.clear() - rounds += 1 - - def _handle_check(self, aauthzrs, index, achalls): - """Returns tuple of ('completed', 'failed').""" - completed = [] - failed = [] - - original_aauthzr = aauthzrs[index] - updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) - aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) - if updated_authzr.body.status == messages.STATUS_VALID: - return achalls, [] - - # Note: if the whole authorization is invalid, the individual failed - # challenges will be determined here... - for achall in achalls: - updated_achall = achall.update(challb=self._find_updated_challb( - updated_authzr, achall)) - - # This does nothing for challenges that have yet to be decided yet. - if updated_achall.status == messages.STATUS_VALID: - completed.append((achall, updated_achall)) - elif updated_achall.status == messages.STATUS_INVALID: - failed.append((achall, updated_achall)) - - return completed, failed - - def _find_updated_challb(self, authzr, achall): # pylint: disable=no-self-use - """Find updated challenge body within Authorization Resource. - - .. warning:: This assumes only one instance of type of challenge in - each challenge resource. - - :param .AuthorizationResource authzr: Authorization Resource - :param .AnnotatedChallenge achall: Annotated challenge for which - to get status - - """ - for authzr_challb in authzr.body.challenges: - if type(authzr_challb.chall) is type(achall.challb.chall): # noqa - return authzr_challb - raise errors.AuthorizationError( - "Target challenge not found in authorization resource") + return achalls def _get_chall_pref(self, domain): """Return list of challenge preferences. @@ -306,43 +209,15 @@ class AuthHandler(object): chall_prefs.extend(plugin_pref) return chall_prefs - def _cleanup_challenges(self, aauthzrs, achalls=None): + def _cleanup_challenges(self, achalls): """Cleanup challenges. - :param aauthzrs: authorizations and their selected annotated - challenges - :type aauthzrs: `list` of `AnnotatedAuthzr` :param achalls: annotated challenges to cleanup :type achalls: `list` of :class:`certbot.achallenges.AnnotatedChallenge` """ logger.info("Cleaning up challenges") - if achalls is None: - achalls = self._get_all_achalls(aauthzrs) - if achalls: - self.auth.cleanup(achalls) - for achall in achalls: - for aauthzr in aauthzrs: - if achall in aauthzr.achalls: - aauthzr.achalls.remove(achall) - break - - def verify_authzr_complete(self, aauthzrs): - """Verifies that all authorizations have been decided. - - :param aauthzrs: authorizations and their selected annotated - challenges - :type aauthzrs: `list` of `AnnotatedAuthzr` - - :returns: Whether all authzr are complete - :rtype: bool - - """ - for aauthzr in aauthzrs: - authzr = aauthzr.authzr - if (authzr.body.status != messages.STATUS_VALID and - authzr.body.status != messages.STATUS_INVALID): - raise errors.AuthorizationError("Incomplete authorizations") + self.auth.cleanup(achalls) def _challenge_factory(self, authzr, path): """Construct Namedtuple Challenges @@ -530,22 +405,19 @@ _ERROR_HELP = { } -def _report_failed_challs(failed_achalls): - """Notifies the user about failed challenges. +def _report_failed_authzrs(failed_authzrs, account_key): + """Notifies the user about failed authorizations.""" + problems = {} # type: Dict[str, List[achallenges.AnnotatedChallenge]] + failed_achalls = [challb_to_achall(challb, account_key, authzr.body.identifier.value) + for authzr in failed_authzrs for challb in authzr.body.challenges + if challb.error] - :param set failed_achalls: A set of failed - :class:`certbot.achallenges.AnnotatedChallenge`. - - """ - problems = collections.defaultdict(list)\ - # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] for achall in failed_achalls: - if achall.error: - problems[achall.error.typ].append(achall) + problems.setdefault(achall.error.typ, []).append(achall) + reporter = zope.component.getUtility(interfaces.IReporter) - for achalls in six.itervalues(problems): - reporter.add_message( - _generate_failed_chall_msg(achalls), reporter.MEDIUM_PRIORITY) + for achalls in problems.values(): + reporter.add_message(_generate_failed_chall_msg(achalls), reporter.MEDIUM_PRIORITY) def _generate_failed_chall_msg(failed_achalls): diff --git a/certbot/interfaces.py b/certbot/interfaces.py index bb9a91b0f..bd91d2272 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -159,21 +159,14 @@ class IAuthenticator(IPlugin): :func:`get_chall_pref` only. :returns: `collections.Iterable` of ACME - :class:`~acme.challenges.ChallengeResponse` instances - or if the :class:`~acme.challenges.Challenge` cannot - be fulfilled then: - - ``None`` - Authenticator can perform challenge, but not at this time. - ``False`` - Authenticator will never be able to perform (error). - + :class:`~acme.challenges.ChallengeResponse` instances corresponding to each provided + :class:`~acme.challenges.Challenge`. :rtype: :class:`collections.Iterable` of :class:`acme.challenges.ChallengeResponse`, where responses are required to be returned in the same order as corresponding input challenges - :raises .PluginError: If challenges cannot be performed + :raises .PluginError: If some or all challenges cannot be performed """ diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index ee1af4978..c14129d87 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -351,7 +351,7 @@ class ChallengePerformer(object): def perform(self): """Perform all added challenges. - :returns: challenge respones + :returns: challenge responses :rtype: `list` of `acme.challenges.KeyAuthorizationChallengeResponse` diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index fe0ece12e..353c34da2 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -4,13 +4,11 @@ import logging import unittest import mock -import six import zope.component from acme import challenges from acme import client as acme_client from acme import messages -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors @@ -82,6 +80,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_account = mock.Mock(key=util.Key("file_path", "PEM")) self.mock_net = mock.MagicMock(spec=acme_client.Client) self.mock_net.acme_version = 1 + self.mock_net.retry_after.side_effect = acme_client.Client.retry_after self.handler = AuthHandler( self.mock_auth, self.mock_net, self.mock_account, []) @@ -95,23 +94,26 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos) mock_order = mock.MagicMock(authorizations=[authzr]) - with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll(retry=1, wait_value=30) + with mock.patch('certbot.auth_handler.time') as mock_time: authzr = self.handler.handle_authorizations(mock_order) - self.assertEqual(self.mock_net.answer_challenge.call_count, 1) + self.assertEqual(self.mock_net.answer_challenge.call_count, 1) - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(list(six.iterkeys(chall_update)), [0]) - self.assertEqual(len(chall_update.values()), 1) + self.assertEqual(self.mock_net.poll.call_count, 2) # Because there is one retry + self.assertEqual(mock_time.sleep.call_count, 2) + # Retry-After header is 30 seconds, but at the time sleep is invoked, several + # instructions are executed, and next pool is in less than 30 seconds. + self.assertTrue(mock_time.sleep.call_args_list[1][0][0] <= 30) + # However, assert that we did not took the default value of 3 seconds. + self.assertTrue(mock_time.sleep.call_args_list[1][0][0] > 3) - self.assertEqual(self.mock_auth.cleanup.call_count, 1) - # Test if list first element is TLSSNI01, use typ because it is an achall - self.assertEqual( - self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + # Test if list first element is TLSSNI01, use typ because it is an achall + self.assertEqual( + self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") - self.assertEqual(len(authzr), 1) + self.assertEqual(len(authzr), 1) def test_name1_tls_sni_01_1_acme_1(self): self._test_name1_tls_sni_01_1_common(combos=True) @@ -120,9 +122,8 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_net.acme_version = 2 self._test_name1_tls_sni_01_1_common(combos=False) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self, mock_poll): - mock_poll.side_effect = self._validate_all + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self): + self.mock_net.poll.side_effect = _gen_mock_on_poll() self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) @@ -132,10 +133,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual(self.mock_net.answer_challenge.call_count, 3) - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(list(six.iterkeys(chall_update)), [0]) - self.assertEqual(len(chall_update.values()), 1) + self.assertEqual(self.mock_net.poll.call_count, 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) # Test if list first element is TLSSNI01, use typ because it is an achall @@ -145,10 +143,9 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p # Length of authorizations list self.assertEqual(len(authzr), 1) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self, mock_poll): + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self): self.mock_net.acme_version = 2 - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll() self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) @@ -158,10 +155,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual(self.mock_net.answer_challenge.call_count, 1) - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(list(six.iterkeys(chall_update)), [0]) - self.assertEqual(len(chall_update.values()), 1) + self.assertEqual(self.mock_net.poll.call_count, 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) cleaned_up_achalls = self.mock_auth.cleanup.call_args[0][0] @@ -175,27 +169,18 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_net.request_domain_challenges.side_effect = functools.partial( gen_dom_authzr, challs=acme_util.CHALLENGES, combos=combos) - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: - mock_poll.side_effect = self._validate_all - authzr = self.handler.handle_authorizations(mock_order) + + self.mock_net.poll.side_effect = _gen_mock_on_poll() + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) # Check poll call - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(len(list(six.iterkeys(chall_update))), 3) - self.assertTrue(0 in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update[0]), 1) - self.assertTrue(1 in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update[1]), 1) - self.assertTrue(2 in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update[2]), 1) + self.assertEqual(self.mock_net.poll.call_count, 3) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -208,14 +193,13 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_net.acme_version = 2 self._test_name3_tls_sni_01_3_common(combos=False) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_debug_challenges(self, mock_poll): + def test_debug_challenges(self): zope.component.provideUtility( mock.Mock(debug_challenges=True), interfaces.IConfig) authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll() self.handler.handle_authorizations(mock_order) @@ -231,6 +215,18 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def test_max_retries_exceeded(self): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + + # We will return STATUS_PENDING twice before returning STATUS_VALID. + self.mock_net.poll.side_effect = _gen_mock_on_poll(retry=2) + + with self.assertRaises(errors.AuthorizationError) as error: + # We retry only once, so retries will be exhausted before STATUS_VALID is returned. + self.handler.handle_authorizations(mock_order, False, 1) + self.assertTrue('All authorizations were not finalized by the CA.' in str(error.exception)) + def test_no_domains(self): mock_order = mock.MagicMock(authorizations=[]) self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) @@ -244,9 +240,8 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: - mock_poll.side_effect = self._validate_all - self.handler.handle_authorizations(mock_order) + self.mock_net.poll.side_effect = _gen_mock_on_poll() + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( @@ -290,11 +285,11 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") - @mock.patch("certbot.auth_handler.AuthHandler._respond") - def test_respond_error(self, mock_respond): + def test_answer_error(self): + self.mock_net.answer_challenge.side_effect = errors.AuthorizationError + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_respond.side_effect = errors.AuthorizationError self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) @@ -302,20 +297,52 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - @mock.patch("certbot.auth_handler.AuthHandler.verify_authzr_complete") - def test_incomplete_authzr_error(self, mock_verify, mock_poll): + def test_incomplete_authzr_error(self): authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_verify.side_effect = errors.AuthorizationError - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll(status=messages.STATUS_INVALID) - self.assertRaises( - errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + with test_util.patch_get_utility(): + with self.assertRaises(errors.AuthorizationError) as error: + self.handler.handle_authorizations(mock_order, False) + self.assertTrue('Some challenges have failed.' in str(error.exception)) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + def test_best_effort(self): + def _conditional_mock_on_poll(authzr): + """This mock will invalidate one authzr, and invalidate the other one""" + valid_mock = _gen_mock_on_poll(messages.STATUS_VALID) + invalid_mock = _gen_mock_on_poll(messages.STATUS_INVALID) + + if authzr.body.identifier.value == 'will-be-invalid': + return invalid_mock(authzr) + return valid_mock(authzr) + + # Two authzrs. Only one will be valid. + authzrs = [gen_dom_authzr(domain="will-be-valid", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="will-be-invalid", challs=acme_util.CHALLENGES)] + self.mock_net.poll.side_effect = _conditional_mock_on_poll + + mock_order = mock.MagicMock(authorizations=authzrs) + + with mock.patch('certbot.auth_handler._report_failed_authzrs') as mock_report: + valid_authzr = self.handler.handle_authorizations(mock_order, True) + + # Because best_effort=True, we did not blow up. Instead ... + self.assertEqual(len(valid_authzr), 1) # ... the valid authzr has been processed + self.assertEqual(mock_report.call_count, 1) # ... the invalid authzr has been reported + + self.mock_net.poll.side_effect = _gen_mock_on_poll(status=messages.STATUS_INVALID) + + with test_util.patch_get_utility(): + with self.assertRaises(errors.AuthorizationError) as error: + self.handler.handle_authorizations(mock_order, True) + + # Despite best_effort=True, process will fail because no authzr is valid. + self.assertTrue('All challenges have failed.' in str(error.exception)) + def test_validated_challenge_not_rerun(self): # With pending challenge, we expect the challenge to be tried, and fail. authzr = acme_util.gen_authzr( @@ -334,138 +361,26 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p mock_order = mock.MagicMock(authorizations=[authzr]) self.handler.handle_authorizations(mock_order) - def _validate_all(self, aauthzrs, unused_1, unused_2): - for i, aauthzr in enumerate(aauthzrs): - azr = aauthzr.authzr - updated_azr = acme_util.gen_authzr( - messages.STATUS_VALID, - azr.body.identifier.value, - [challb.chall for challb in azr.body.challenges], - [messages.STATUS_VALID] * len(azr.body.challenges), - azr.body.combinations) - aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) - @mock.patch("certbot.auth_handler.logger") def test_tls_sni_logs(self, logger): self._test_name1_tls_sni_01_1_common(combos=True) self.assertTrue("deprecated" in logger.warning.call_args[0][0]) -class PollChallengesTest(unittest.TestCase): - # pylint: disable=protected-access - """Test poll challenges.""" +def _gen_mock_on_poll(status=messages.STATUS_VALID, retry=0, wait_value=1): + state = {'count': retry} - def setUp(self): - from certbot.auth_handler import challb_to_achall - from certbot.auth_handler import AuthHandler, AnnotatedAuthzr - - # Account and network are mocked... - self.mock_net = mock.MagicMock() - self.handler = AuthHandler( - None, self.mock_net, mock.Mock(key="mock_key"), []) - - self.doms = ["0", "1", "2"] - self.aauthzrs = [ - AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[0], - [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False), []), - AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []), - AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []) - ] - - self.chall_update = {} # type: Dict[int, achallenges.KeyAuthorizationAnnotatedChallenge] - for i, aauthzr in enumerate(self.aauthzrs): - self.chall_update[i] = [ - challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) - for challb in aauthzr.authzr.body.challenges] - - - @mock.patch("certbot.auth_handler.time") - def test_poll_challenges(self, unused_mock_time): - self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.handler._poll_challenges(self.aauthzrs, self.chall_update, False) - - for aauthzr in self.aauthzrs: - self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) - - @mock.patch("certbot.auth_handler.time") - def test_poll_challenges_failure_best_effort(self, unused_mock_time): - self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid - self.handler._poll_challenges(self.aauthzrs, self.chall_update, True) - - for aauthzr in self.aauthzrs: - self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) - - @mock.patch("certbot.auth_handler.time") - @test_util.patch_get_utility() - def test_poll_challenges_failure(self, unused_mock_time, unused_mock_zope): - self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid - self.assertRaises( - errors.AuthorizationError, self.handler._poll_challenges, - self.aauthzrs, self.chall_update, False) - - @mock.patch("certbot.auth_handler.time") - def test_unable_to_find_challenge_status(self, unused_mock_time): - from certbot.auth_handler import challb_to_achall - self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.chall_update[0].append( - challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) - self.assertRaises( - errors.AuthorizationError, self.handler._poll_challenges, - self.aauthzrs, self.chall_update, False) - - def test_verify_authzr_failure(self): - self.assertRaises(errors.AuthorizationError, - self.handler.verify_authzr_complete, self.aauthzrs) - - def _mock_poll_solve_one_valid(self, authzr): - # Pending here because my dummy script won't change the full status. - # Basically it didn't raise an error and it stopped earlier than - # Making all challenges invalid which would make mock_poll_solve_one - # change authzr to invalid - return self._mock_poll_solve_one_chall(authzr, messages.STATUS_VALID) - - def _mock_poll_solve_one_invalid(self, authzr): - return self._mock_poll_solve_one_chall(authzr, messages.STATUS_INVALID) - - def _mock_poll_solve_one_chall(self, authzr, desired_status): - # pylint: disable=no-self-use - """Dummy method that solves one chall at a time to desired_status. - - When all are solved.. it changes authzr.status to desired_status - - """ - new_challbs = authzr.body.challenges - for challb in authzr.body.challenges: - if challb.status != desired_status: - new_challbs = tuple( - challb_temp if challb_temp != challb - else acme_util.chall_to_challb(challb.chall, desired_status) - for challb_temp in authzr.body.challenges - ) - break - - if all(test_challb.status == desired_status - for test_challb in new_challbs): - status_ = desired_status - else: - status_ = authzr.body.status - - new_authzr = messages.AuthorizationResource( - uri=authzr.uri, - body=messages.Authorization( - identifier=authzr.body.identifier, - challenges=new_challbs, - combinations=authzr.body.combinations, - status=status_, - ), - ) - return (new_authzr, "response") + def _mock(authzr): + state['count'] = state['count'] - 1 + effective_status = status if state['count'] < 0 else messages.STATUS_PENDING + updated_azr = acme_util.gen_authzr( + effective_status, + authzr.body.identifier.value, + [challb.chall for challb in authzr.body.challenges], + [effective_status] * len(authzr.body.challenges), + authzr.body.combinations) + return updated_azr, mock.MagicMock(headers={'Retry-After': str(wait_value)}) + return _mock class ChallbToAchallTest(unittest.TestCase): @@ -527,8 +442,8 @@ class GenChallengePathTest(unittest.TestCase): errors.AuthorizationError, self._call, challbs, prefs, None) -class ReportFailedChallsTest(unittest.TestCase): - """Tests for certbot.auth_handler._report_failed_challs.""" +class ReportFailedAuthzrsTest(unittest.TestCase): + """Tests for certbot.auth_handler._report_failed_authzrs.""" # pylint: disable=protected-access def setUp(self): @@ -542,31 +457,27 @@ class ReportFailedChallsTest(unittest.TestCase): # Prevent future regressions if the error type changes self.assertTrue(kwargs["error"].description is not None) - self.http01 = achallenges.KeyAuthorizationAnnotatedChallenge( - # pylint: disable=star-args - challb=messages.ChallengeBody(**kwargs), - domain="example.com", - account_key="key") + http_01 = messages.ChallengeBody(**kwargs) # pylint: disable=star-args kwargs["chall"] = acme_util.TLSSNI01 - self.tls_sni_same = achallenges.KeyAuthorizationAnnotatedChallenge( - # pylint: disable=star-args - challb=messages.ChallengeBody(**kwargs), - domain="example.com", - account_key="key") + tls_sni_01 = messages.ChallengeBody(**kwargs) # pylint: disable=star-args + + self.authzr1 = mock.MagicMock() + self.authzr1.body.identifier.value = 'example.com' + self.authzr1.body.challenges = [http_01, tls_sni_01] kwargs["error"] = messages.Error(typ="dnssec", detail="detail") - self.tls_sni_diff = achallenges.KeyAuthorizationAnnotatedChallenge( - # pylint: disable=star-args - challb=messages.ChallengeBody(**kwargs), - domain="foo.bar", - account_key="key") + tls_sni_01_diff = messages.ChallengeBody(**kwargs) # pylint: disable=star-args + + self.authzr2 = mock.MagicMock() + self.authzr2.body.identifier.value = 'foo.bar' + self.authzr2.body.challenges = [tls_sni_01_diff] @test_util.patch_get_utility() def test_same_error_and_domain(self, mock_zope): from certbot import auth_handler - auth_handler._report_failed_challs([self.http01, self.tls_sni_same]) + auth_handler._report_failed_authzrs([self.authzr1], 'key') call_list = mock_zope().add_message.call_args_list self.assertTrue(len(call_list) == 1) self.assertTrue("Domain: example.com\nType: tls\nDetail: detail" in call_list[0][0][0]) @@ -575,7 +486,7 @@ class ReportFailedChallsTest(unittest.TestCase): def test_different_errors_and_domains(self, mock_zope): from certbot import auth_handler - auth_handler._report_failed_challs([self.http01, self.tls_sni_diff]) + auth_handler._report_failed_authzrs([self.authzr1, self.authzr2], 'key') self.assertTrue(mock_zope().add_message.call_count == 2) diff --git a/tox.cover.py b/tox.cover.py index e323ba255..d0f97626a 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -12,7 +12,7 @@ DEFAULT_PACKAGES = [ 'certbot_dns_sakuracloud', 'certbot_nginx', 'certbot_postfix', 'letshelp_certbot'] COVER_THRESHOLDS = { - 'certbot': {'linux': 98, 'windows': 94}, + 'certbot': {'linux': 98, 'windows': 93}, 'acme': {'linux': 100, 'windows': 99}, 'certbot_apache': {'linux': 100, 'windows': 100}, 'certbot_dns_cloudflare': {'linux': 98, 'windows': 98}, From bf1f83f47b49f746f1cd54076749a08e7b04250d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 2 Mar 2019 02:16:22 +0100 Subject: [PATCH 26/44] Revert "Disable build for commits pushed on master (#6804)" (#6807) By removing all the builds on push to master, done in #6804, we also removing the coveralls reports that are necessary to calculate the effect of a PR on code coverage, that is part of the quality gate process. This PR is reverting #6804, and another implementation preserving the coveralls reports will be done soon. This reverts commit a468a3b255e326dbfa2b7d629eb20fa275631254. --- .travis.yml | 5 ----- appveyor.yml | 8 -------- 2 files changed, 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 108f853aa..4adab1e4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,11 +18,6 @@ branches: - /^\d+\.\d+\.x$/ - /^test-.*$/ -# Since master can receive only commits from PR that have already been tested, we avoid with the -# the following condition to launch again a pipeline when the merge commit is pushed to master. -# However master still needs to be set in branches section to allow PR for master to be built. -if: NOT (type = push AND branch = master) - matrix: include: # These environments are always executed diff --git a/appveyor.yml b/appveyor.yml index 5dcc80ba9..2b6b82747 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,14 +11,6 @@ branches: - /^\d+\.\d+\.x$/ # Version branches like X.X.X - /^test-.*$/ -init: - # Since master can receive only commits from PR that have already been tested, we avoid with the - # the following condition to launch again a pipeline when the merge commit is pushed to master. - # However master still needs to be set in branches section to allow PR for master to be built. - - ps: | - if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master') - { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild } - install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" From 6ff101dcbbe66943487633391a4f03f27bcc6aaa Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 2 Mar 2019 03:07:07 +0100 Subject: [PATCH 27/44] Cover case of OpenSUSE Leap 15+ in certbot-auto (#6794) Fixes #6228. Since OpenSUSE Leap 15, python-virtualenv became a source package, breaking certbot-auto bootstrap on this version. Then python2-virtualenv must be used to create Python 2.x virtual environments. This PR makes certbot-auto compatible to prior and after Leap 15, by testing the existence of python-virtualenv on current OpenSUSE system, and then use appropriate packages. * Cover case of OpenSUSE Leap 15+ in certbot-auto * Revert increment on bootstrap for OpenSUSE * Fix configuration for Leap15+ * Add comment about explicit installation of python2-setuptools * Update letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh Co-Authored-By: adferrand * Update letsencrypt-auto --- letsencrypt-auto-source/letsencrypt-auto | 12 +++++++++++- .../pieces/bootstrappers/suse_common.sh | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index c390f28ec..ce3b35e86 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -521,10 +521,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ diff --git a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh index c531cbe99..ac66119c3 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh @@ -14,10 +14,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ From d8a3fa3904ecddd00be871847c773c9370780449 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 4 Mar 2019 15:52:38 +0200 Subject: [PATCH 28/44] Address review comments --- tools/merge_requirements.py | 25 ++++++++++++++----------- tools/pip_install.py | 2 +- tools/strip_hashes.py | 33 +++++++++++++++++++-------------- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index 521d28b8f..0d41d12c4 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -55,21 +55,24 @@ def main(*paths): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier - files. + files. Files are read from file paths passed from the command line arguments. - :param tuple paths: paths to requirements files + If no command line arguments are defined, data is read from stdin instead. + + :param tuple paths: paths to requirements files provided on command line """ data = {} - for path in paths: - data.update(process_entries(read_file(path))) - - # Need to check if interactive to avoid blocking if nothing is piped - if not sys.stdin.isatty(): - stdin_data = [] - for line in sys.stdin: - stdin_data.append(line) - data.update(process_entries(stdin_data)) + if paths: + for path in paths: + data.update(process_entries(read_file(path))) + else: + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + data.update(process_entries(stdin_data)) return output_requirements(data) diff --git a/tools/pip_install.py b/tools/pip_install.py index 8f0437d9c..15dc2f0c0 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -48,7 +48,7 @@ def certbot_normal_processing(tools_path, test_constraints): with open(certbot_requirements, 'r') as fd: data = fd.readlines() with open(test_constraints, 'w') as fd: - data = os.linesep.join(strip_hashes.process_entries(data)) + data = "\n".join(strip_hashes.process_entries(data)) fd.write(data) diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py index 7e7809458..988e72eb8 100755 --- a/tools/strip_hashes.py +++ b/tools/strip_hashes.py @@ -2,7 +2,6 @@ """Removes hash information from requirement files passed to it as file path arguments or simply piped to stdin.""" -import os import re import sys @@ -24,22 +23,28 @@ def process_entries(entries): return out_lines def main(*paths): - """Reads dependency definitions from a (list of) file(s) or stdin and - removes hashes from returned entries""" + """ + Reads dependency definitions from a (list of) file(s) provided on the + command line. If no command line arguments are present, data is read from + stdin instead. + + Hashes are removed from returned entries. + """ deps = [] - for path in paths: - with open(path) as file_h: - deps += process_entries(file_h.readlines()) + if paths: + for path in paths: + with open(path) as file_h: + deps += process_entries(file_h.readlines()) + else: + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + deps += process_entries(stdin_data) - # Need to check if interactive to avoid blocking if nothing is piped - if not sys.stdin.isatty(): - stdin_data = [] - for line in sys.stdin: - stdin_data.append(line) - deps += process_entries(stdin_data) - - return os.linesep.join(deps) + return "\n".join(deps) if __name__ == '__main__': print(main(*sys.argv[1:])) # pylint: disable=star-args From 6a0f3248a8f7a55b2de33283e498458e3a7b1a81 Mon Sep 17 00:00:00 2001 From: Yohann Leon Date: Tue, 5 Mar 2019 22:27:07 +0100 Subject: [PATCH 29/44] Replace deprecated Gandi plugin link --- docs/using.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index e17e56b64..de17742a1 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -277,7 +277,7 @@ Plugin Auth Inst Notes plesk_ Y Y Integration with the Plesk web hosting tool haproxy_ Y Y Integration with the HAProxy load balancer s3front_ Y Y Integration with Amazon CloudFront distribution of S3 buckets -gandi_ Y Y Integration with Gandi's hosting products and API +gandi_ Y Y Integration with Gandi LiveDNS API varnish_ Y N Obtain certificates via a Varnish server external_ Y N A plugin for convenient scripting (See also ticket 2782_) icecast_ N Y Deploy certificates to Icecast 2 streaming media servers @@ -290,7 +290,7 @@ heroku_ Y Y Integration with Heroku SSL .. _plesk: https://github.com/plesk/letsencrypt-plesk .. _haproxy: https://github.com/greenhost/certbot-haproxy .. _s3front: https://github.com/dlapiduz/letsencrypt-s3front -.. _gandi: https://github.com/Gandi/letsencrypt-gandi +.. _gandi: https://github.com/obynio/certbot-plugin-gandi .. _icecast: https://github.com/e00E/lets-encrypt-icecast .. _varnish: http://git.sesse.net/?p=letsencrypt-varnish-plugin .. _2782: https://github.com/certbot/certbot/issues/2782 From 198c447e77c785842a68c2fdc9326f156f29d915 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 5 Mar 2019 14:01:08 -0800 Subject: [PATCH 30/44] Move the OCSP change to the right section. (#6818) Looks like this got added to our changelog for the released 0.31.0 instead of the upcoming release. We want this change for the release tomorrow. --- CHANGELOG.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 764fc0946..ce0dddd16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added -* +* If possible, Certbot uses built-in support for OCSP from recent cryptography + versions instead of the OpenSSL binary: as a consequence Certbot does not need + the OpenSSL binary to be installed anymore if cryptography>=2.5 is installed. ### Changed @@ -49,9 +51,6 @@ More details about these changes can be found on our GitHub repo. * Avoid reprocessing challenges that are already validated when a certificate is issued. -* If possible, Certbot uses built-in support for OCSP from recent cryptography - versions instead of the OpenSSL binary: as a consequence Certbot does not need - the OpenSSL binary to be installed anymore if cryptography>=2.5 is installed. * Support for initiating (but not solving end-to-end) TLS-ALPN-01 challenges with the `acme` module. From 670e9d89b70258a1a8fb963f4e208de0e22aab6d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 6 Mar 2019 03:30:33 +0100 Subject: [PATCH 31/44] Fix faulty test (#6816) --- .../tests/apache-conf-files/passing/finalize-1243.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf index 0918e5669..dbfae3765 100644 --- a/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf @@ -1,7 +1,7 @@ #LoadModule ssl_module modules/mod_ssl.so -Listen 443 - +Listen 4443 + # The ServerName directive sets the request scheme, hostname and port that # the server uses to identify itself. This is used when creating # redirection URLs. In the context of virtual hosts, the ServerName From be6df7de044988613ff14ac042cda3a7e863b076 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 6 Mar 2019 12:16:13 -0800 Subject: [PATCH 32/44] Remove Fixed section from changelog --- CHANGELOG.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce0dddd16..cc007b46f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,10 +30,6 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). field included if a `malformed` error is received. This fallback will be removed in version 0.34.0. -### Fixed - -* - Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: From a276523c096bfa46c5948b24e37cccb0d6a2bd25 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 6 Mar 2019 12:18:08 -0800 Subject: [PATCH 33/44] Update changelog for 0.32.0 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc007b46f..fdb3b5f13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 0.32.0 - master +## 0.32.0 - 2019-03-06 ### Added From 0492855166f104bba8a7dd0395b925d82c8cc759 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 6 Mar 2019 12:47:27 -0800 Subject: [PATCH 34/44] Release 0.32.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 100 ++++++++++-------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 10 +- letsencrypt-auto | 100 ++++++++++-------- letsencrypt-auto-source/certbot-auto.asc | 16 +-- letsencrypt-auto-source/letsencrypt-auto | 26 ++--- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++--- 26 files changed, 168 insertions(+), 146 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 6cb5c3f92..00e9a0971 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.32.0.dev0' +version = '0.32.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 5d15611dd..1b3173ada 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools.command.test import test as TestCommand import sys -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 03c2994e3..0c82a7437 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.31.0" +LE_AUTO_VERSION="0.32.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -521,10 +521,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ @@ -1034,26 +1044,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1180,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.31.0 \ - --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ - --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 -acme==0.31.0 \ - --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ - --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf -certbot-apache==0.31.0 \ - --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ - --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd -certbot-nginx==0.31.0 \ - --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ - --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 +certbot==0.32.0 \ + --hash=sha256:75fd986ae42cd90bde6400c5f5a0dd936a7f4a42a416146b1e8bb0f92028b443 \ + --hash=sha256:c0b94e25a07d83809d98029f09e9b501f86ec97624f45ce86800a7002488c3c8 +acme==0.32.0 \ + --hash=sha256:88b2d2741e5ea028c590a33b16fb647cb74af6b2db6c7909c738a48f879efdec \ + --hash=sha256:0eefce8b7880eb7eccc049a6b8ba262fc624bc34b3a8581d05b82f2bb39f1aec +certbot-apache==0.32.0 \ + --hash=sha256:b2c82b7a1c44799ba3a150970513ed4fa9afeee40e326440800b1243f917ddb6 \ + --hash=sha256:68072775f1bb4bc9fc64cabe051a761f6dbf296012512eff7819144ac8b9ec97 +certbot-nginx==0.32.0 \ + --hash=sha256:3fc3664231586565d886ddcb679c95a2fb2494a2ce3e028149f1496dca5b47cf \ + --hash=sha256:82c43cd26aacc2eb0ae890be6a2f74d726b6dcb4ee7b63c0e55ec33e576f3e84 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1221,7 +1231,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -1241,7 +1250,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1263,7 +1272,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -1344,7 +1353,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -1354,12 +1364,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -1372,7 +1382,7 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e1c82b063..c11e32cd1 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d6a16f468..8c697d0a5 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index f0f2446e1..ed3ac8c78 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index c61f05339..f9d2c430f 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 7d39a10d0..2039d27c7 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 8a5f234b5..6df71672f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 67ae1d4eb..3aee64ba4 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 429702d75..e7ff6ac78 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 6797501b9..60f6e0d36 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 07d9decdf..4406244fd 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 314b0a16c..bdd21f45b 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index a878eac47..046c05382 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 39f35e953..580f4ac2a 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index e915c6b86..24e097b9d 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 429f960f7..8c6e28c90 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.32.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 8984704a6..b6cd0ea97 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools.command.test import test as TestCommand import sys -version = '0.32.0.dev0' +version = '0.32.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 767448a12..a442d1e84 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.32.0.dev0' +__version__ = '0.32.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index e95b2fcd5..ed7794c3b 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -113,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.31.0 + "". (default: CertbotACMEClient/0.32.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -477,7 +477,9 @@ plugins: using Sakura Cloud for DNS). (default: False) apache: - Apache Web Server plugin + Apache Web Server plugin (Please note that the default values of the + Apache plugin options change depending on the operating system Certbot is + run on.) --apache-enmod APACHE_ENMOD Path to the Apache 'a2enmod' binary (default: None) @@ -496,7 +498,7 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2/other) + /etc/apache2) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you (Only Ubuntu/Debian currently) (default: False) @@ -505,7 +507,7 @@ apache: Ubuntu/Debian currently) (default: False) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apachectl) + apache2ctl) dns-cloudflare: Obtain certificates using a DNS TXT record (if you are using Cloudflare diff --git a/letsencrypt-auto b/letsencrypt-auto index 03c2994e3..0c82a7437 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.31.0" +LE_AUTO_VERSION="0.32.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -521,10 +521,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ @@ -1034,26 +1044,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1180,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.31.0 \ - --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ - --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 -acme==0.31.0 \ - --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ - --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf -certbot-apache==0.31.0 \ - --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ - --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd -certbot-nginx==0.31.0 \ - --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ - --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 +certbot==0.32.0 \ + --hash=sha256:75fd986ae42cd90bde6400c5f5a0dd936a7f4a42a416146b1e8bb0f92028b443 \ + --hash=sha256:c0b94e25a07d83809d98029f09e9b501f86ec97624f45ce86800a7002488c3c8 +acme==0.32.0 \ + --hash=sha256:88b2d2741e5ea028c590a33b16fb647cb74af6b2db6c7909c738a48f879efdec \ + --hash=sha256:0eefce8b7880eb7eccc049a6b8ba262fc624bc34b3a8581d05b82f2bb39f1aec +certbot-apache==0.32.0 \ + --hash=sha256:b2c82b7a1c44799ba3a150970513ed4fa9afeee40e326440800b1243f917ddb6 \ + --hash=sha256:68072775f1bb4bc9fc64cabe051a761f6dbf296012512eff7819144ac8b9ec97 +certbot-nginx==0.32.0 \ + --hash=sha256:3fc3664231586565d886ddcb679c95a2fb2494a2ce3e028149f1496dca5b47cf \ + --hash=sha256:82c43cd26aacc2eb0ae890be6a2f74d726b6dcb4ee7b63c0e55ec33e576f3e84 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1221,7 +1231,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -1241,7 +1250,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1263,7 +1272,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -1344,7 +1353,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -1354,12 +1364,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -1372,7 +1382,7 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index d95f9abcb..eeb8d7d12 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlxcop8ACgkQTRfJlc2X -dfIbZwf/faKu7IjLi0qFQ+kw8zaAnV47JDgfWqbR5GSdwWPqld+QyHlcRfPgwYma -fKj9+g/FvPNPSfjHRRCoFrYvpZ4lZ+f4HPN9+OjydfM77rdDhVDwzs8dbKIk02yU -0IEJhXj5Q9hF3TSDZcyXAJdBU1lz51ohtVIXelMBPmzhYPCZF47iE9/k9pApQi86 -RTji7hxPcF/n7mzXrbyTvk+kDxSdDlE0Eg9syK7XaFDBTa2lqgG8wTnMPVqhc/hm -WM/uwkzbYarjy05ffV1kM683nP0rECnHlYT38pYcT2puw2kn/QthwR5j/jB/DWSc -94Kw7BeMH651V8EaNwYIiouylnVH3A== -=U+Qh +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlyAMZ0ACgkQTRfJlc2X +dfLItQf/SNv+at1Pw5oiEbWleNPpmz9srlkf9AHU92Hh3p7+OljcaWQindtCYtlO +UvWV/CjGzObmJvO+Pgy2epNhD8cTjPamI46l5UG2nwvy8V+JemS937Ae6paivt8T +/RaFKfyNDfxBjQhHS1ypVuRrFgAQ5CG0iGuJSMgwLpcKCZyKAim3+Vb57+Esq+zG +Cp7GmJk9h1z5FbNukbaFHBlJQIefJoQclh1yUw11pLab0uxIOdc9WiEWLLAVE512 +SMQM2sNv49uh7mRnxW+6WU6dor6JI9Ff1L4D5hfglzBJRM4qLU/hGv54oVycWq4i +eFjtqDfo5XMwnbnUnVkEB73pY6lzDg== +=YaE3 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index ce3b35e86..0c82a7437 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.32.0.dev0" +LE_AUTO_VERSION="0.32.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1190,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.31.0 \ - --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ - --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 -acme==0.31.0 \ - --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ - --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf -certbot-apache==0.31.0 \ - --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ - --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd -certbot-nginx==0.31.0 \ - --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ - --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 +certbot==0.32.0 \ + --hash=sha256:75fd986ae42cd90bde6400c5f5a0dd936a7f4a42a416146b1e8bb0f92028b443 \ + --hash=sha256:c0b94e25a07d83809d98029f09e9b501f86ec97624f45ce86800a7002488c3c8 +acme==0.32.0 \ + --hash=sha256:88b2d2741e5ea028c590a33b16fb647cb74af6b2db6c7909c738a48f879efdec \ + --hash=sha256:0eefce8b7880eb7eccc049a6b8ba262fc624bc34b3a8581d05b82f2bb39f1aec +certbot-apache==0.32.0 \ + --hash=sha256:b2c82b7a1c44799ba3a150970513ed4fa9afeee40e326440800b1243f917ddb6 \ + --hash=sha256:68072775f1bb4bc9fc64cabe051a761f6dbf296012512eff7819144ac8b9ec97 +certbot-nginx==0.32.0 \ + --hash=sha256:3fc3664231586565d886ddcb679c95a2fb2494a2ce3e028149f1496dca5b47cf \ + --hash=sha256:82c43cd26aacc2eb0ae890be6a2f74d726b6dcb4ee7b63c0e55ec33e576f3e84 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 3ecca151022170a559f05135a0b3ec86bc998e96..17eea3eba1b349826997251971e17ac8efe7ea52 100644 GIT binary patch literal 256 zcmV+b0ssCf-^}x|Ewd+i;#v>piH?o-`-qDCn+vf z$3RtG;n%M3^4xW@~i8c0kFB$Jt-(u9-Ju}xO!+YXSFfC2&HlmJy zAhH1>F_F9cSN3V5Z%-0cd~Hf{3?0JP`3%W9Iyz30B3!F$a}q6xTC|sDM>me@1oCdpGMSAJlPA0}OyX15pk1>`;NPfsRIoA7989)8hXTrsI)?K2n+{4(K0$wW- z?Xt|aQ|bMz{LJj09XOy^>A@XGdSH*2KX{+1-dg&>BqH~3rK zY2Wn@jjzL2vZ@UiY;SJVRrZ9^Fl|&kDuWGZ&Hbdn^_ Date: Wed, 6 Mar 2019 12:47:28 -0800 Subject: [PATCH 35/44] Add contents to CHANGELOG.md for next version --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdb3b5f13..c854c74bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 0.33.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo. + ## 0.32.0 - 2019-03-06 ### Added From 8dda6cc68f7f2a0b2b05d223e27df487fb5c9b3c Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 6 Mar 2019 12:47:29 -0800 Subject: [PATCH 36/44] Bump version to 0.33.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/local-oldest-requirements.txt | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 00e9a0971..cd4ea3ef5 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.32.0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 1b3173ada..dfedc3f0d 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools.command.test import test as TestCommand import sys -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index c11e32cd1..858cef831 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 8c697d0a5..84414e0c0 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index ed3ac8c78..4b2fe15be 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index f9d2c430f..1de51f7ca 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 2039d27c7..113e55e12 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 6df71672f..6e882a4f7 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 3aee64ba4..bdbd198d1 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index e7ff6ac78..c2556797d 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 60f6e0d36..c494d6d4e 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4406244fd..4c57c5709 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index bdd21f45b..9c9f233cc 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 046c05382..cf6fee5d7 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 580f4ac2a..dacf41101 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 24e097b9d..321ad3b5d 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 8c6e28c90..46d92a6ff 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index db6b261f0..2108298ae 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ acme[dev]==0.29.0 --e .[dev] +certbot[dev]==0.32.0 diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b6cd0ea97..7494bf6bf 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools.command.test import test as TestCommand import sys -version = '0.32.0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index a442d1e84..38e25f3a4 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.32.0' +__version__ = '0.33.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0c82a7437..af4b181cd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.32.0" +LE_AUTO_VERSION="0.33.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From f378536ffab6213d1e34a8811e9d8a0e6ab9674b Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 6 Mar 2019 23:49:43 +0100 Subject: [PATCH 37/44] Do not run the full CI pipeline on master (#6811) * Configure appveyor * Renaming in travis yml * Update .travis.yml Co-Authored-By: adferrand * Update .travis.yml Co-Authored-By: adferrand --- .travis.yml | 68 ++++++++++++++++++++++++++++++++++------------------ appveyor.yml | 8 +++++++ 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4adab1e4e..cabd3cf73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,141 +18,163 @@ branches: - /^\d+\.\d+\.x$/ - /^test-.*$/ +# Jobs for the main test suite are always executed (including on PRs) except for pushes on master. +not-on-master: ¬-on-master + if: NOT (type = push AND branch = master) + +# Jobs for the extended test suite are executed for cron jobs and pushes on non-master branches. +extended-test-suite: &extended-test-suite + if: type = cron OR (type = push AND branch != master) + matrix: include: - # These environments are always executed + # Main test suite - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker + <<: *not-on-master - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker + <<: *not-on-master + + # This job is always executed, including on master - python: "2.7" env: TOXENV=py27-cover FYI="py27 tests + code coverage" + - sudo: required env: TOXENV=nginx_compat services: docker before_install: addons: + <<: *not-on-master - python: "2.7" env: TOXENV=lint + <<: *not-on-master - python: "3.4" env: TOXENV=mypy + <<: *not-on-master - python: "3.5" env: TOXENV=mypy + <<: *not-on-master - python: "2.7" env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest' sudo: required services: docker + <<: *not-on-master - python: "3.4" env: TOXENV=py34 sudo: required services: docker + <<: *not-on-master - python: "3.7" dist: xenial env: TOXENV=py37 sudo: required services: docker + <<: *not-on-master - sudo: required env: TOXENV=apache_compat services: docker before_install: addons: + <<: *not-on-master - sudo: required env: TOXENV=le_auto_trusty services: docker before_install: addons: + <<: *not-on-master - python: "2.7" env: TOXENV=apacheconftest-with-pebble sudo: required services: docker + <<: *not-on-master - python: "2.7" env: TOXENV=nginxroundtrip + <<: *not-on-master - # These environments are executed on cron events and commits to tested - # branches other than master. Which branches are tested is controlled by - # the "branches" section earlier in this file. + # Extended test suite on cron jobs and pushes to tested branches other than master - python: "3.7" dist: xenial env: TOXENV=py37 CERTBOT_NO_PIN=1 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.4" env: TOXENV=py34 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.4" env: TOXENV=py34 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.5" env: TOXENV=py35 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.5" env: TOXENV=py35 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.6" env: TOXENV=py36 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.6" env: TOXENV=py36 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.7" dist: xenial env: TOXENV=py37 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.7" dist: xenial env: TOXENV=py37 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=le_auto_xenial services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=le_auto_jessie services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=le_auto_centos6 services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=docker_dev services: docker @@ -160,7 +182,7 @@ matrix: apt: packages: # don't install nginx and apache - libaugeas0 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - language: generic env: TOXENV=py27 os: osx @@ -169,7 +191,7 @@ matrix: packages: - augeas - python2 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - language: generic env: TOXENV=py3 os: osx @@ -178,7 +200,7 @@ matrix: packages: - augeas - python3 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite # container-based infrastructure sudo: false diff --git a/appveyor.yml b/appveyor.yml index 2b6b82747..12d882973 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,6 +11,14 @@ branches: - /^\d+\.\d+\.x$/ # Version branches like X.X.X - /^test-.*$/ +init: + # Since master can receive only commits from PR that have already been tested, following + # condition avoid to launch all jobs except the coverage one for commits pushed to master. + - ps: | + if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master' ` + -And -Not ($Env:TOXENV -Like '*-cover')) + { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild } + install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" From 34393f9bf4a0ca7045a4cfd1e4eafa5842b3a832 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 7 Mar 2019 19:05:20 +0100 Subject: [PATCH 38/44] Correct certbot-auto for Fedora 29+ (#6812) Fixes #6698 Fedora maintainers engaged a deprecation path for Python 2.x with Fedora 29. As a first step, python2-virtualenv does not install the virtualenv binary anymore, in favor of python3-virtualenv, and so the installation of Python 3 virtual environments by default. However, certbot-auto installs python2-virtualenv for all recent RPM distributions, and relies of the execution of virtualenv, and this is failing the process. Since the plan in the future is to remove Python 2.x from Fedora, this PR follows this logic to fix certbot-auto: started to Fedora 29, certbot-auto will install and execute certbot on Python 3. This implies to detect that we are on Fedora 29+, install python3-virtualenv that will install also Python 3 dependencies and virtualenv binary, then instruct the process to use Python 3. This is in fact similar to EOL distributions shipping with Python 2.6, and for which Python 3.4 from EPEL is installed and used. Older versions of Fedora continue to use Python 2.x, and their process is untouched. Four scenarios are covered here: fresh Fedora 28: old process is used, nothing changes fresh Fedora 29: new process is used, Python 3 is installed, certbot runs on it update Fedora 29 from 28, already installed certbot-auto without rebootstrapping required: existing venv continue to be used, certbot runs on it update Fedora 29 from 28, already installed certbot-auto with rebootstrapping required: new process is used, installing python3-virtualenv, python3-devel and python3-rpm-macros, Python 3 is installed, certbot runs on it * Add a step to handle python3 on fedora29 * Update letsencrypt-auto-source/letsencrypt-auto.template Co-Authored-By: adferrand * Update letsencrypt-auto-source/letsencrypt-auto.template Co-Authored-By: adferrand * Update letsencrypt-auto-source/letsencrypt-auto.template Co-Authored-By: adferrand * Update rpm_python3.sh * Rebuild certbot-auto * Empty commit to relaunch CI pipeline * Add changelog * Update CHANGELOG.md Co-Authored-By: adferrand * Update CHANGELOG.md --- CHANGELOG.md | 3 ++- letsencrypt-auto-source/letsencrypt-auto | 14 ++++++++++++-- letsencrypt-auto-source/letsencrypt-auto.template | 5 ++++- .../pieces/bootstrappers/rpm_python3.sh | 9 ++++++++- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c854c74bf..3b49d5275 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added -* +* Fedora 29+ is now supported by certbot-auto. Since Python 2.x is on a deprecation + path in Fedora, certbot-auto will install and use Python 3.x on Fedora 29+. ### Changed diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index af4b181cd..df7fc2dc1 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -488,11 +488,18 @@ BOOTSTRAP_RPM_PYTHON3_VERSION=1 BootstrapRpmPython3() { # Tested with: # - CentOS 6 + # - Fedora 29 InitializeRPMCommonBase + # Fedora 29 must use python3-virtualenv + if $TOOL list python3-virtualenv >/dev/null 2>&1; then + python_pkgs="python3 + python3-virtualenv + python3-devel + " # EPEL uses python34 - if $TOOL list python34 >/dev/null 2>&1; then + elif $TOOL list python34 >/dev/null 2>&1; then python_pkgs="python34 python34-devel python34-tools @@ -741,7 +748,10 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then + # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. + RPM_DIST_NAME=`(. /etc/os-release && echo $ID) || echo "unknown"` + RPM_DIST_VERSION=`(. /etc/os-release && echo $VERSION_ID) || echo "0"` + if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then Bootstrap() { BootstrapMessage "RedHat-based OSes that will use Python3" BootstrapRpmPython3 diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2cd3f4336..215fca04e 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -323,7 +323,10 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then + # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. + RPM_DIST_NAME=`(. /etc/os-release && echo $ID) || echo "unknown"` + RPM_DIST_VERSION=`(. /etc/os-release && echo $VERSION_ID) || echo "0"` + if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then Bootstrap() { BootstrapMessage "RedHat-based OSes that will use Python3" BootstrapRpmPython3 diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh index b011a7235..f33b07ca9 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh @@ -5,11 +5,18 @@ BOOTSTRAP_RPM_PYTHON3_VERSION=1 BootstrapRpmPython3() { # Tested with: # - CentOS 6 + # - Fedora 29 InitializeRPMCommonBase + # Fedora 29 must use python3-virtualenv + if $TOOL list python3-virtualenv >/dev/null 2>&1; then + python_pkgs="python3 + python3-virtualenv + python3-devel + " # EPEL uses python34 - if $TOOL list python34 >/dev/null 2>&1; then + elif $TOOL list python34 >/dev/null 2>&1; then python_pkgs="python34 python34-devel python34-tools From 45229eebdfd08baac26c9e8b78d76490b6f7cbce Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Mar 2019 11:57:08 -0800 Subject: [PATCH 39/44] Drop expected Apache coverage to workaround #6813. (#6826) * Drop expected Apache coverage to workaround #6813. * add comment --- tox.cover.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.cover.py b/tox.cover.py index d0f97626a..6f5392b71 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -14,7 +14,10 @@ DEFAULT_PACKAGES = [ COVER_THRESHOLDS = { 'certbot': {'linux': 98, 'windows': 93}, 'acme': {'linux': 100, 'windows': 99}, - 'certbot_apache': {'linux': 100, 'windows': 100}, + # certbot_apache coverage not being at 100% is a workaround for + # https://github.com/certbot/certbot/issues/6813. We should increase + # the minimum coverage back to 100% when this issue is resolved. + 'certbot_apache': {'linux': 99, 'windows': 99}, 'certbot_dns_cloudflare': {'linux': 98, 'windows': 98}, 'certbot_dns_cloudxns': {'linux': 99, 'windows': 99}, 'certbot_dns_digitalocean': {'linux': 98, 'windows': 98}, From cac4be7046e7c349fd4f03ae8af51b7586de4138 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 11 Mar 2019 23:27:33 +0100 Subject: [PATCH 40/44] Calculate timedelta with thisUpdate/nextUpdate in UTC (#6838) Fixes #6836. OCSP responses contains a thisUpdate and nextUpdate that allow to calculate its validity. Certbot currently uses datetime.now() to get the current time when OCSP check is done through cryptography. But datetime.now() expresses the date in the machine local time, and comparison operators on datetime do not take into account the offset between two datetime objects expressed in difference timezones. As a consequence, a given thisUpdate may be seen as a future date depending on the local time, failing the OCSP check process. The error is not critical for certbot, as it will just make some valid OCSP responses giving an EXPIRED status been ignored. This PR fixes this comparison by taking the current time in UTC using datetime.utctime(). --- certbot/ocsp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/ocsp.py b/certbot/ocsp.py index 6f4c0b0fb..bc63c40f9 100644 --- a/certbot/ocsp.py +++ b/certbot/ocsp.py @@ -200,7 +200,7 @@ def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert): # for OpenSSL, so we do not do it here. # See OpenSSL implementation as a reference: # https://github.com/openssl/openssl/blob/ef45aa14c5af024fcb8bef1c9007f3d1c115bd85/crypto/ocsp/ocsp_cl.c#L338-L391 - now = datetime.now() + now = datetime.utcnow() # thisUpdate/nextUpdate are expressed in UTC/GMT time zone if not response_ocsp.this_update: raise AssertionError('param thisUpdate is not set.') if response_ocsp.this_update > now + timedelta(minutes=5): From 81d9b5250e1d50b376170186b661c6c4ca132956 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 11 Mar 2019 23:42:32 +0100 Subject: [PATCH 41/44] Clean stderr in case of /etc/os-release does not exist (#6835) --- letsencrypt-auto-source/letsencrypt-auto | 4 ++-- letsencrypt-auto-source/letsencrypt-auto.template | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index df7fc2dc1..2a44a58dc 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -749,8 +749,8 @@ elif [ -f /etc/redhat-release ]; then unset LE_PYTHON DeterminePythonVersion "NOCRASH" # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. - RPM_DIST_NAME=`(. /etc/os-release && echo $ID) || echo "unknown"` - RPM_DIST_VERSION=`(. /etc/os-release && echo $VERSION_ID) || echo "0"` + RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"` + RPM_DIST_VERSION=`(. /etc/os-release 2> /dev/null && echo $VERSION_ID) || echo "0"` if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then Bootstrap() { BootstrapMessage "RedHat-based OSes that will use Python3" diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 215fca04e..29c54bf2d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -324,8 +324,8 @@ elif [ -f /etc/redhat-release ]; then unset LE_PYTHON DeterminePythonVersion "NOCRASH" # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. - RPM_DIST_NAME=`(. /etc/os-release && echo $ID) || echo "unknown"` - RPM_DIST_VERSION=`(. /etc/os-release && echo $VERSION_ID) || echo "0"` + RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"` + RPM_DIST_VERSION=`(. /etc/os-release 2> /dev/null && echo $VERSION_ID) || echo "0"` if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then Bootstrap() { BootstrapMessage "RedHat-based OSes that will use Python3" From e20adedb94fdf6774ad3789e6f493905d56f750f Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 11 Mar 2019 15:50:27 -0700 Subject: [PATCH 42/44] This is now at the top level of their site --- docs/ciphers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ciphers.rst b/docs/ciphers.rst index 1b320cdf9..b748dd87a 100644 --- a/docs/ciphers.rst +++ b/docs/ciphers.rst @@ -227,7 +227,7 @@ BetterCrypto.org BetterCrypto.org, a collaboration of mostly European IT security experts, has published a draft paper, "Applied Crypto Hardening" -https://bettercrypto.org/static/applied-crypto-hardening.pdf +https://bettercrypto.org/ FF-DHE Internet-Draft ~~~~~~~~~~~~~~~~~~~~~ From a7f2f24426e7d4025e5c32b684324a3fb8a81f01 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 11 Mar 2019 16:14:20 -0700 Subject: [PATCH 43/44] Mention OCSP UTC fix in changelog. (#6845) --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b49d5275..cc34634ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,13 +15,15 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* +* Certbot uses the Python library cryptography for OCSP when cryptography>=2.5 + is installed. We fixed a bug in Certbot causing it to interpret timestamps in + the OCSP response as being in the local timezone rather than UTC. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: -* +* certbot More details about these changes can be found on our GitHub repo. From cf29e89366c7eb57dd48a99a06fe05ceaa9057fa Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 12 Mar 2019 00:16:48 +0100 Subject: [PATCH 44/44] Move coverage computation during certbot integration tests at the end of the script (#6842) Currently coverage invocation during integration tests on certbot core is misplaced, just before the OCSP statuses tests. This PR move back the coverage invocation at the end of the script. --- tests/certbot-boulder-integration.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 9d1ae9ec2..05b144df4 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -528,8 +528,6 @@ if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then --manual-cleanup-hook ./tests/manual-dns-cleanup.sh fi -coverage report --fail-under 64 --include 'certbot/*' --show-missing - # Test OCSP status ## OCSP 1: Check stale OCSP status @@ -582,3 +580,5 @@ if [ "$REVOKED" != 1 ] ; then echo "Expected le-ocsp-check.wtf to be REVOKED" exit 1 fi + +coverage report --fail-under 64 --include 'certbot/*' --show-missing