From 00634394f2e077ec5e621acd869e370f8619cd08 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Jan 2018 21:16:44 -0800 Subject: [PATCH 01/98] Only respect LE_PYTHON inside USE_PYTHON_3 if we know a user must have set it version 2 (#5402) * stop exporting LE_PYTHON * unset LE_PYTHON sometimes --- letsencrypt-auto-source/letsencrypt-auto | 8 ++++++-- letsencrypt-auto-source/letsencrypt-auto.template | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 712ef6813..0ad089acd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -274,7 +274,6 @@ DeterminePythonVersion() { return 0 fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt "$MIN_PYVER" ]; then @@ -801,7 +800,7 @@ elif [ -f /etc/redhat-release ]; then } BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi - export LE_PYTHON="$prev_le_python" + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -906,6 +905,10 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" if [ -d "$VENV_PATH" ]; then # If the selected Bootstrap function isn't a noop and it differs from the @@ -1412,6 +1415,7 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index b06ac9c80..5b8b1c164 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -274,7 +274,6 @@ DeterminePythonVersion() { return 0 fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt "$MIN_PYVER" ]; then @@ -340,7 +339,7 @@ elif [ -f /etc/redhat-release ]; then } BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi - export LE_PYTHON="$prev_le_python" + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -445,6 +444,10 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" if [ -d "$VENV_PATH" ]; then # If the selected Bootstrap function isn't a noop and it differs from the @@ -571,6 +574,7 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then From 3acf5d1ef909b2e775588a7e045bb00728e2de83 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 10 Jan 2018 12:10:21 -0800 Subject: [PATCH 02/98] Fix rebootstraping with old venvs (#5392) * Fix rebootstrapping before venv move * add regression test * dedupe test * Cleanup case when two venvs exist. * Add clarifying comment * Add double venv test to leauto_upgrades * Fix logic with the help of coffee * redirect stderr * pass VENV_PATH through sudo * redirect stderr --- letsencrypt-auto-source/letsencrypt-auto | 24 ++++++++++-- .../letsencrypt-auto.template | 24 ++++++++++-- .../letstest/scripts/test_leauto_upgrades.sh | 37 ++++++++++++++++++- 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0ad089acd..b86517bb6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -897,7 +897,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -910,13 +914,21 @@ if [ "$1" = "--le-auto-phase2" ]; then fi INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -926,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -1418,7 +1434,7 @@ else export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 5b8b1c164..96e5c2db0 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -436,7 +436,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -449,13 +453,21 @@ if [ "$1" = "--le-auto-phase2" ]; then fi INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -465,6 +477,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -577,7 +593,7 @@ else export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index a83cbd826..51472f2e6 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -64,10 +64,45 @@ iQIDAQAB -----END PUBLIC KEY----- " -if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then +if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then + if command -v python3; then + echo "Didn't expect Python 3 to be installed!" + exit 1 + fi + cp letsencrypt-auto cb-auto + if ! ./cb-auto -v --debug --version 2>&1 | grep 0.5.0 ; then + echo "Certbot shouldn't have updated to a new version!" + exit 1 + fi + if [ -d "/opt/eff.org" ]; then + echo "New directory shouldn't have been created!" + exit 1 + fi + # Create a 2nd venv at the new path to ensure we properly handle this case + export VENV_PATH="/opt/eff.org/certbot/venv" + if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then + echo second installation appeared to fail + exit 1 + fi + unset VENV_PATH + EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) + if ! ./cb-auto -v --debug --version -n 2>&1 | grep "$EXPECTED_VERSION" ; then + echo "Certbot didn't upgrade as expected!" + exit 1 + fi + if ! command -v python3; then + echo "Python3 wasn't properly installed" + exit 1 + fi + if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1)" != 3 ]; then + echo "Python3 wasn't used in venv!" + exit 1 + fi +elif ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then echo upgrade appeared to fail exit 1 fi + echo upgrade appeared to be successful if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then From 39472f88dead13782f2d84e65135cf11b96258bb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 10 Jan 2018 13:26:31 -0800 Subject: [PATCH 03/98] reduce ipdb version (#5408) --- tools/dev_constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index afc362ff8..47440241d 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -21,7 +21,7 @@ futures==3.1.1 google-api-python-client==1.5 httplib2==0.10.3 imagesize==0.7.1 -ipdb==0.10.3 +ipdb==0.10.2 ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 From 9e952081014b9545ce6ddb8b6ecc86a51bf94131 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 10 Jan 2018 18:34:45 -0800 Subject: [PATCH 04/98] Factor out common challengeperformer logic (#5413) --- .../certbot_apache/tests/tls_sni_01_test.py | 4 +- .../certbot_nginx/tests/tls_sni_01_test.py | 2 +- certbot/plugins/common.py | 42 ++++++++++++---- certbot/plugins/common_test.py | 48 ++++++++++++------- 4 files changed, 67 insertions(+), 29 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py index 6c37c2ecc..42fb3021b 100644 --- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py +++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py @@ -16,8 +16,8 @@ from six.moves import xrange # pylint: disable=redefined-builtin, import-error class TlsSniPerformTest(util.ApacheTest): """Test the ApacheTlsSni01 challenge.""" - auth_key = common_test.TLSSNI01Test.auth_key - achalls = common_test.TLSSNI01Test.achalls + auth_key = common_test.AUTH_KEY + achalls = common_test.ACHALLS def setUp(self): # pylint: disable=arguments-differ super(TlsSniPerformTest, self).setUp() diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 32a5ed7d2..61ee293fa 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -20,7 +20,7 @@ from certbot_nginx.tests import util class TlsSniPerformTest(util.NginxTest): """Test the NginxTlsSni01 challenge.""" - account_key = common_test.TLSSNI01Test.auth_key + account_key = common_test.AUTH_KEY achalls = [ achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 002d2f225..c281534ca 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -315,23 +315,28 @@ class Addr(object): return result -class TLSSNI01(object): - """Abstract base for TLS-SNI-01 challenge performers""" +class ChallengePerformer(object): + """Abstract base for challenge performers. + + :ivar configurator: Authenticator and installer plugin + :ivar achalls: Annotated challenges + :vartype achalls: `list` of `.KeyAuthorizationAnnotatedChallenge` + :ivar indices: Holds the indices of challenges from a larger array + so the user of the class doesn't have to. + :vartype indices: `list` of `int` + + """ def __init__(self, configurator): self.configurator = configurator self.achalls = [] self.indices = [] - self.challenge_conf = os.path.join( - configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf") - # self.completed = 0 def add_chall(self, achall, idx=None): - """Add challenge to TLSSNI01 object to perform at once. + """Store challenge to be performed when perform() is called. :param .KeyAuthorizationAnnotatedChallenge achall: Annotated - TLSSNI01 challenge. - + challenge. :param int idx: index to challenge in a larger array """ @@ -339,6 +344,27 @@ class TLSSNI01(object): if idx is not None: self.indices.append(idx) + def perform(self): + """Perform all added challenges. + + :returns: challenge respones + :rtype: `list` of `acme.challenges.KeyAuthorizationChallengeResponse` + + + """ + raise NotImplementedError() + + +class TLSSNI01(ChallengePerformer): + # pylint: disable=abstract-method + """Abstract base for TLS-SNI-01 challenge performers""" + + def __init__(self, configurator): + super(TLSSNI01, self).__init__(configurator) + self.challenge_conf = os.path.join( + configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf") + # self.completed = 0 + def get_cert_path(self, achall): """Returns standardized name for challenge certificate. diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 1a1ca7dcb..103a12499 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -18,6 +18,17 @@ from certbot import errors from certbot.tests import acme_util from certbot.tests import util as test_util +AUTH_KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) +ACHALLS = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.TLSSNI01(token=b'token1'), "pending"), + domain="encryption-example.demo", account_key=AUTH_KEY), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.TLSSNI01(token=b'token2'), "pending"), + domain="certbot.demo", account_key=AUTH_KEY), +] class NamespaceFunctionsTest(unittest.TestCase): """Tests for certbot.plugins.common.*_namespace functions.""" @@ -261,21 +272,27 @@ class AddrTest(unittest.TestCase): self.assertEqual(set_c, set_d) +class ChallengePerformerTest(unittest.TestCase): + """Tests for certbot.plugins.common.ChallengePerformer.""" + + def setUp(self): + configurator = mock.MagicMock() + + from certbot.plugins.common import ChallengePerformer + self.performer = ChallengePerformer(configurator) + + def test_add_chall(self): + self.performer.add_chall(ACHALLS[0], 0) + self.assertEqual(1, len(self.performer.achalls)) + self.assertEqual([0], self.performer.indices) + + def test_perform(self): + self.assertRaises(NotImplementedError, self.performer.perform) + + class TLSSNI01Test(unittest.TestCase): """Tests for certbot.plugins.common.TLSSNI01.""" - auth_key = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) - achalls = [ - achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.TLSSNI01(token=b'token1'), "pending"), - domain="encryption-example.demo", account_key=auth_key), - achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.TLSSNI01(token=b'token2'), "pending"), - domain="certbot.demo", account_key=auth_key), - ] - def setUp(self): self.tempdir = tempfile.mkdtemp() configurator = mock.MagicMock() @@ -288,11 +305,6 @@ class TLSSNI01Test(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) - def test_add_chall(self): - self.sni.add_chall(self.achalls[0], 0) - self.assertEqual(1, len(self.sni.achalls)) - self.assertEqual([0], self.sni.indices) - def test_setup_challenge_cert(self): # This is a helper function that can be used for handling # open context managers more elegantly. It avoids dealing with @@ -325,7 +337,7 @@ class TLSSNI01Test(unittest.TestCase): OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)) def test_get_z_domain(self): - achall = self.achalls[0] + achall = ACHALLS[0] self.assertEqual(self.sni.get_z_domain(achall), achall.response(achall.account_key).z_domain.decode("utf-8")) From 2ba334a18202926e4e6b4f166a18457861bba031 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 10 Jan 2018 20:14:56 -0800 Subject: [PATCH 05/98] Add basic HTTP01 support to Apache * Add a simple version of HTTP01 * remove cert from chall name * make directory work on 2.2 * cleanup challenges when finished * import shutil * fixup perform and cleanup tests * Add tests for http_01.py --- certbot-apache/certbot_apache/configurator.py | 36 ++++-- certbot-apache/certbot_apache/http_01.py | 94 +++++++++++++++ .../certbot_apache/tests/configurator_test.py | 70 +++++++---- .../certbot_apache/tests/http_01_test.py | 112 ++++++++++++++++++ .../certbot_apache/tests/tls_sni_01_test.py | 2 +- 5 files changed, 278 insertions(+), 36 deletions(-) create mode 100644 certbot-apache/certbot_apache/http_01.py create mode 100644 certbot-apache/certbot_apache/tests/http_01_test.py diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 5a33346ea..60441f30c 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -24,9 +24,10 @@ from certbot_apache import apache_util from certbot_apache import augeas_configurator from certbot_apache import constants from certbot_apache import display_ops -from certbot_apache import tls_sni_01 +from certbot_apache import http_01 from certbot_apache import obj from certbot_apache import parser +from certbot_apache import tls_sni_01 from collections import defaultdict @@ -163,6 +164,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "ensure-http-header": self._set_http_header, "staple-ocsp": self._enable_ocsp_stapling} + # This will be set during the perform function + self.http_doer = None + @property def mod_ssl_conf(self): """Full absolute path to SSL configuration file.""" @@ -1855,7 +1859,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.TLSSNI01] + return [challenges.TLSSNI01, challenges.HTTP01] def perform(self, achalls): """Perform the configuration related challenge. @@ -1867,16 +1871,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ self._chall_out.update(achalls) responses = [None] * len(achalls) - chall_doer = tls_sni_01.ApacheTlsSni01(self) + self.http_doer = http_01.ApacheHttp01(self) + sni_doer = tls_sni_01.ApacheTlsSni01(self) for i, achall in enumerate(achalls): # Currently also have chall_doer hold associated index of the # challenge. This helps to put all of the responses back together # when they are all complete. - chall_doer.add_chall(achall, i) + if isinstance(achall.chall, challenges.HTTP01): + self.http_doer.add_chall(achall, i) + else: # tls-sni-01 + sni_doer.add_chall(achall, i) - sni_response = chall_doer.perform() - if sni_response: + http_response = self.http_doer.perform() + sni_response = sni_doer.perform() + if http_response or sni_response: # Must reload in order to activate the challenges. # Handled here because we may be able to load up other challenge # types @@ -1886,14 +1895,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # of identifying when the new configuration is being used. time.sleep(3) - # Go through all of the challenges and assign them to the proper - # place in the responses return value. All responses must be in the - # same order as the original challenges. - for i, resp in enumerate(sni_response): - responses[chall_doer.indices[i]] = resp + self._update_responses(responses, http_response, self.http_doer) + self._update_responses(responses, sni_response, sni_doer) return responses + def _update_responses(self, responses, chall_response, chall_doer): + # Go through all of the challenges and assign them to the proper + # place in the responses return value. All responses must be in the + # same order as the original challenges. + for i, resp in enumerate(chall_response): + responses[chall_doer.indices[i]] = resp + def cleanup(self, achalls): """Revert all challenges.""" self._chall_out.difference_update(achalls) @@ -1903,6 +1916,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.revert_challenge_config() self.restart() self.parser.reset_modules() + self.http_doer.cleanup() def install_ssl_options_conf(self, options_ssl, options_ssl_digest): """Copy Certbot's SSL options file into the system's config dir if required.""" diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py new file mode 100644 index 000000000..410e8e9a6 --- /dev/null +++ b/certbot-apache/certbot_apache/http_01.py @@ -0,0 +1,94 @@ +"""A class that performs HTTP-01 challenges for Apache""" +import logging +import os +import shutil +import tempfile + +from certbot.plugins import common + +logger = logging.getLogger(__name__) + +class ApacheHttp01(common.TLSSNI01): + """Class that performs HTPP-01 challenges within the Apache configurator.""" + + CONFIG_TEMPLATE24 = """\ +Alias /.well-known/acme-challenge {0} + + + Require all granted + + +""" + + CONFIG_TEMPLATE22 = """\ +Alias /.well-known/acme-challenge {0} + + + Order allow,deny + Allow from all + + +""" + + def __init__(self, *args, **kwargs): + super(ApacheHttp01, self).__init__(*args, **kwargs) + self.challenge_conf = os.path.join( + self.configurator.conf("challenge-location"), + "le_http_01_challenge.conf") + self.challenge_dir = None + + def perform(self): + """Perform all HTTP-01 challenges.""" + if not self.achalls: + return [] + # Save any changes to the configuration as a precaution + # About to make temporary changes to the config + self.configurator.save("Changes before challenge setup", True) + + responses = self._set_up_challenges() + self._mod_config() + # Save reversible changes + self.configurator.save("HTTP Challenge", True) + + return responses + + def cleanup(self): + """Cleanup the challenge directory.""" + shutil.rmtree(self.challenge_dir, ignore_errors=True) + self.challenge_dir = None + + def _mod_config(self): + self.configurator.parser.add_include( + self.configurator.parser.loc["default"], self.challenge_conf) + self.configurator.reverter.register_file_creation( + True, self.challenge_conf) + + if self.configurator.version < (2, 4): + config_template = self.CONFIG_TEMPLATE22 + else: + config_template = self.CONFIG_TEMPLATE24 + config_text = config_template.format(self.challenge_dir) + + logger.debug("writing a config file with text:\n %s", config_text) + with open(self.challenge_conf, "w") as new_conf: + new_conf.write(config_text) + + def _set_up_challenges(self): + self.challenge_dir = tempfile.mkdtemp() + os.chmod(self.challenge_dir, 0o755) + + responses = [] + for achall in self.achalls: + responses.append(self._set_up_challenge(achall)) + + return responses + + def _set_up_challenge(self, achall): + response, validation = achall.response_and_validation() + + name = os.path.join(self.challenge_dir, achall.chall.encode("token")) + with open(name, 'wb') as f: + f.write(validation.encode()) + os.chmod(name, 0o644) + + return response diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 4f85e1e3f..1620013c8 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -676,23 +676,33 @@ class MultipleVhostsTest(util.ApacheTest): self.config._add_name_vhost_if_necessary(self.vh_truth[0]) self.assertEqual(self.config.add_name_vhost.call_count, 2) + @mock.patch("certbot_apache.configurator.http_01.ApacheHttp01.perform") @mock.patch("certbot_apache.configurator.tls_sni_01.ApacheTlsSni01.perform") @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") - def test_perform(self, mock_restart, mock_perform): + def test_perform(self, mock_restart, mock_tls_perform, mock_http_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded - account_key, achall1, achall2 = self.get_achalls() + account_key, achalls = self.get_key_and_achalls() - expected = [ - achall1.response(account_key), - achall2.response(account_key), - ] + all_expected = [] + http_expected = [] + tls_expected = [] + for achall in achalls: + response = achall.response(account_key) + if isinstance(achall.chall, challenges.HTTP01): + http_expected.append(response) + else: + tls_expected.append(response) + all_expected.append(response) - mock_perform.return_value = expected - responses = self.config.perform([achall1, achall2]) + mock_http_perform.return_value = http_expected + mock_tls_perform.return_value = tls_expected - self.assertEqual(mock_perform.call_count, 1) - self.assertEqual(responses, expected) + responses = self.config.perform(achalls) + + self.assertEqual(mock_http_perform.call_count, 1) + self.assertEqual(mock_tls_perform.call_count, 1) + self.assertEqual(responses, all_expected) self.assertEqual(mock_restart.call_count, 1) @@ -700,30 +710,38 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") def test_cleanup(self, mock_cfg, mock_restart): mock_cfg.return_value = "" - _, achall1, achall2 = self.get_achalls() + _, achalls = self.get_key_and_achalls() + self.config.http_doer = mock.MagicMock() - self.config._chall_out.add(achall1) # pylint: disable=protected-access - self.config._chall_out.add(achall2) # pylint: disable=protected-access + for achall in achalls: + self.config._chall_out.add(achall) # pylint: disable=protected-access - self.config.cleanup([achall1]) - self.assertFalse(mock_restart.called) - - self.config.cleanup([achall2]) - self.assertTrue(mock_restart.called) + for i, achall in enumerate(achalls): + self.config.cleanup([achall]) + if i == len(achalls) - 1: + self.assertTrue(mock_restart.called) + self.assertTrue(self.config.http_doer.cleanup.called) + else: + self.assertFalse(mock_restart.called) + self.assertFalse(self.config.http_doer.cleanup.called) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") def test_cleanup_no_errors(self, mock_cfg, mock_restart): mock_cfg.return_value = "" - _, achall1, achall2 = self.get_achalls() + _, achalls = self.get_key_and_achalls() + self.config.http_doer = mock.MagicMock() - self.config._chall_out.add(achall1) # pylint: disable=protected-access + for achall in achalls: + self.config._chall_out.add(achall) # pylint: disable=protected-access - self.config.cleanup([achall2]) + self.config.cleanup([achalls[-1]]) self.assertFalse(mock_restart.called) + self.assertFalse(self.config.http_doer.cleanup.called) - self.config.cleanup([achall1, achall2]) + self.config.cleanup(achalls) self.assertTrue(mock_restart.called) + self.assertTrue(self.config.http_doer.cleanup.called) @mock.patch("certbot.util.run_script") def test_get_version(self, mock_script): @@ -1151,7 +1169,7 @@ class MultipleVhostsTest(util.ApacheTest): not_rewriterule = "NotRewriteRule ^ ..." self.assertFalse(self.config._sift_rewrite_rule(not_rewriterule)) - def get_achalls(self): + def get_key_and_achalls(self): """Return testing achallenges.""" account_key = self.rsa512jwk achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( @@ -1166,8 +1184,12 @@ class MultipleVhostsTest(util.ApacheTest): token=b"uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"), "pending"), domain="certbot.demo", account_key=account_key) + achall3 = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=(b'x' * 16)), "pending"), + domain="example.org", account_key=account_key) - return account_key, achall1, achall2 + return account_key, (achall1, achall2, achall3) def test_make_addrs_sni_ready(self): self.config.version = (2, 2) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py new file mode 100644 index 000000000..11121fae2 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -0,0 +1,112 @@ +"""Test for certbot_apache.http_01.""" +import os +import unittest + +from acme import challenges + +from certbot import achallenges + +from certbot.tests import acme_util + +from certbot_apache.tests import util + + +NUM_ACHALLS = 3 + + +class ApacheHttp01TestMeta(type): + """Generates parmeterized tests for testing perform.""" + def __new__(mcs, name, bases, class_dict): + + def _gen_test(num_achalls, minor_version): + def _test(self): + achalls = self.achalls[:num_achalls] + self.config.version = (2, minor_version) + self.common_perform_test(achalls) + return _test + + for i in range(1, NUM_ACHALLS + 1): + for j in (2, 4): + test_name = "test_perform_{0}_{1}".format(i, j) + class_dict[test_name] = _gen_test(i, j) + return type.__new__(mcs, name, bases, class_dict) + + +class ApacheHttp01Test(util.ApacheTest): + """Test for certbot_apache.http_01.ApacheHttp01.""" + + __metaclass__ = ApacheHttp01TestMeta + + def setUp(self, *args, **kwargs): + super(ApacheHttp01Test, self).setUp(*args, **kwargs) + self.maxDiff = None + + self.account_key = self.rsa512jwk + self.achalls = [] + for i in range(NUM_ACHALLS): + self.achalls.append( + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((chr(ord('a') + i) * 16))), + "pending"), + domain="example{0}.com".format(i), + account_key=self.account_key)) + + from certbot_apache.http_01 import ApacheHttp01 + self.http = ApacheHttp01(self.config) + + def test_empty_perform(self): + self.assertFalse(self.http.perform()) + + def common_perform_test(self, achalls): + """Tests perform with the given achalls.""" + for achall in achalls: + self.http.add_chall(achall) + + expected_response = [ + achall.response(self.account_key) for achall in achalls] + self.assertEqual(self.http.perform(), expected_response) + + self.assertTrue(os.path.isdir(self.http.challenge_dir)) + self._has_min_permissions(self.http.challenge_dir, 0o755) + self._test_challenge_conf() + + for achall in achalls: + self._test_challenge_file(achall) + + challenge_dir = self.http.challenge_dir + self.http.cleanup() + self.assertFalse(os.path.exists(challenge_dir)) + + def _test_challenge_conf(self): + self.assertEqual( + len(self.config.parser.find_dir( + "Include", self.http.challenge_conf)), 1) + + with open(self.http.challenge_conf) as f: + conf_contents = f.read() + + alias_fmt = "Alias /.well-known/acme-challenge {0}" + alias = alias_fmt.format(self.http.challenge_dir) + self.assertTrue(alias in conf_contents) + if self.config.version < (2, 4): + self.assertTrue("Allow from all" in conf_contents) + else: + self.assertTrue("Require all granted" in conf_contents) + + def _test_challenge_file(self, achall): + name = os.path.join(self.http.challenge_dir, achall.chall.encode("token")) + validation = achall.validation(self.account_key) + + self._has_min_permissions(name, 0o644) + with open(name, 'rb') as f: + self.assertEqual(f.read(), validation.encode()) + + def _has_min_permissions(self, path, min_mode): + """Tests the given file has at least the permissions in mode.""" + st_mode = os.stat(path).st_mode + self.assertEqual(st_mode, st_mode | min_mode) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py index 42fb3021b..8cea97f04 100644 --- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py +++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py @@ -1,6 +1,6 @@ """Test for certbot_apache.tls_sni_01.""" -import unittest import shutil +import unittest import mock From fa97877cfb058a8a6eda0e86b079876230bcfb7d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 11 Jan 2018 14:46:48 +0200 Subject: [PATCH 06/98] Make sure that Apache is listening on port 80 and has mod_alias * Ensure that mod_alias is enabled * Make sure we listen to port http01_port --- certbot-apache/certbot_apache/configurator.py | 62 +++++++++++++++---- certbot-apache/certbot_apache/http_01.py | 11 ++++ .../certbot_apache/tests/configurator_test.py | 38 +++++++++++- .../certbot_apache/tests/http_01_test.py | 11 ++++ certbot-apache/certbot_apache/tests/util.py | 1 + 5 files changed, 111 insertions(+), 12 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 60441f30c..3b91617e5 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -740,28 +740,40 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ - # If nonstandard port, add service definition for matching - if port != "443": + self.prepare_https_modules(temp) + self.ensure_listen(port, https=True) + + def ensure_listen(self, port, https=False): + """Make sure that Apache is listening on the port. Checks if the + Listen statement for the port already exists, and adds it to the + configuration if necessary. + + :param str port: Port number to check and add Listen for if not in + place already + :param bool https: If the port will be used for HTTPS + + """ + + # If HTTPS requested for nonstandard port, add service definition + if https and port != "443": port_service = "%s %s" % (port, "https") else: port_service = port - self.prepare_https_modules(temp) # Check for Listen # Note: This could be made to also look for ip:443 combo listens = [self.parser.get_arg(x).split()[0] for x in self.parser.find_dir("Listen")] - # In case no Listens are set (which really is a broken apache config) - if not listens: - listens = ["80"] - # Listen already in place if self._has_port_already(listens, port): return listen_dirs = set(listens) + if not listens: + listen_dirs.add(port_service) + for listen in listens: # For any listen statement, check if the machine also listens on # Port 443. If not, add such a listen statement. @@ -776,11 +788,39 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if "%s:%s" % (ip, port_service) not in listen_dirs and ( "%s:%s" % (ip, port_service) not in listen_dirs): listen_dirs.add("%s:%s" % (ip, port_service)) - self._add_listens(listen_dirs, listens, port) + if https: + self._add_listens_https(listen_dirs, listens, port) + else: + self._add_listens_http(listen_dirs, listens, port) - def _add_listens(self, listens, listens_orig, port): - """Helper method for prepare_server_https to figure out which new - listen statements need adding + def _add_listens_http(self, listens, listens_orig, port): + """Helper method for ensure_listen to figure out which new + listen statements need adding for listening HTTP on port + + :param set listens: Set of all needed Listen statements + :param list listens_orig: List of existing listen statements + :param string port: Port number we're adding + """ + + new_listens = listens.difference(listens_orig) + + if port in new_listens: + # We have wildcard, skip the rest + self.parser.add_dir(parser.get_aug_path(self.parser.loc["listen"]), + "Listen", port) + self.save_notes += "Added Listen %s directive to %s\n" % ( + port, self.parser.loc["listen"]) + else: + for listen in new_listens: + self.parser.add_dir(parser.get_aug_path( + self.parser.loc["listen"]), "Listen", listen.split(" ")) + self.save_notes += ("Added Listen %s directive to " + "%s\n") % (listen, + self.parser.loc["listen"]) + + def _add_listens_https(self, listens, listens_orig, port): + """Helper method for ensure_listen to figure out which new + listen statements need adding for listening HTTPS on port :param set listens: Set of all needed Listen statements :param list listens_orig: List of existing listen statements diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 410e8e9a6..9c25b1f17 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -45,6 +45,10 @@ Alias /.well-known/acme-challenge {0} # About to make temporary changes to the config self.configurator.save("Changes before challenge setup", True) + self.configurator.ensure_listen(str( + self.configurator.config.http01_port)) + self.prepare_http01_modules() + responses = self._set_up_challenges() self._mod_config() # Save reversible changes @@ -57,6 +61,13 @@ Alias /.well-known/acme-challenge {0} shutil.rmtree(self.challenge_dir, ignore_errors=True) self.challenge_dir = None + def prepare_http01_modules(self): + """Make sure that we have the needed modules available for http01""" + + if self.configurator.conf("handle-modules"): + if "alias_module" not in self.configurator.parser.modules: + self.configurator.enable_mod("alias", temp=True) + def _mod_config(self): self.configurator.parser.add_include( self.configurator.parser.loc["default"], self.challenge_conf) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 1620013c8..ea3867061 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -424,6 +424,43 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue(self.config.parser.find_dir( "NameVirtualHost", "*:80")) + def test_add_listen_80(self): + mock_find = mock.Mock() + mock_add_dir = mock.Mock() + mock_find.return_value = [] + self.config.parser.find_dir = mock_find + self.config.parser.add_dir = mock_add_dir + self.config.ensure_listen("80") + self.assertTrue(mock_add_dir.called) + self.assertTrue(mock_find.called) + self.assertEqual(mock_add_dir.call_args[0][1], "Listen") + self.assertEqual(mock_add_dir.call_args[0][2], "80") + + def test_add_listen_80_named(self): + mock_find = mock.Mock() + mock_find.return_value = ["test1", "test2", "test3"] + mock_get = mock.Mock() + mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"] + mock_add_dir = mock.Mock() + + self.config.parser.find_dir = mock_find + self.config.parser.get_arg = mock_get + self.config.parser.add_dir = mock_add_dir + + self.config.ensure_listen("80") + self.assertEqual(mock_add_dir.call_count, 0) + + # Reset return lists and inputs + mock_add_dir.reset_mock() + mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"] + + # Test + self.config.ensure_listen("8080") + self.assertEqual(mock_add_dir.call_count, 3) + self.assertTrue(mock_add_dir.called) + self.assertEqual(mock_add_dir.call_args[0][1], "Listen") + self.assertEqual(mock_add_dir.call_args[0][2], ['1.2.3.4:8080']) + def test_prepare_server_https(self): mock_enable = mock.Mock() self.config.enable_mod = mock_enable @@ -435,7 +472,6 @@ class MultipleVhostsTest(util.ApacheTest): # This will test the Add listen self.config.parser.find_dir = mock_find self.config.parser.add_dir_to_ifmodssl = mock_add_dir - self.config.prepare_server_https("443") # Changing the order these modules are enabled breaks the reverter self.assertEqual(mock_enable.call_args_list[0][0][0], "socache_shmcb") diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 11121fae2..f52b9d0cc 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -1,4 +1,5 @@ """Test for certbot_apache.http_01.""" +import mock import os import unittest @@ -53,11 +54,21 @@ class ApacheHttp01Test(util.ApacheTest): account_key=self.account_key)) from certbot_apache.http_01 import ApacheHttp01 + self.config.parser.modules.add("mod_alias.c") + self.config.parser.modules.add("alias_module") self.http = ApacheHttp01(self.config) def test_empty_perform(self): self.assertFalse(self.http.perform()) + @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") + def test_add_alias_module(self, mock_enmod): + self.config.parser.modules.remove("alias_module") + self.config.parser.modules.remove("mod_alias.c") + self.http.prepare_http01_modules() + self.assertTrue(mock_enmod.called) + self.assertEqual(mock_enmod.call_args[0][0], "alias") + def common_perform_test(self, achalls): """Tests perform with the given achalls.""" for achall in achalls: diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index ca667465c..8f5162629 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -103,6 +103,7 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc apache_challenge_location=config_path, backup_dir=backups, config_dir=config_dir, + http01_port="80", temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), in_progress_dir=os.path.join(backups, "IN_PROGRESS"), work_dir=work_dir) From f0f5defb6ff9fcc54db4bbc977950610b22cd155 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 11 Jan 2018 09:27:30 -0800 Subject: [PATCH 07/98] Address minor concerns with Apache HTTP-01 * enable other modules * change port type * remove maxDiff from test class * update port comment * add -f to a2dismod --- certbot-apache/certbot_apache/configurator.py | 2 +- certbot-apache/certbot_apache/http_01.py | 10 ++++- .../certbot_apache/override_debian.py | 2 +- .../certbot_apache/tests/http_01_test.py | 39 ++++++++++++++++--- certbot-apache/certbot_apache/tests/util.py | 2 +- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 3b91617e5..8d6995211 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -776,7 +776,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for listen in listens: # For any listen statement, check if the machine also listens on - # Port 443. If not, add such a listen statement. + # the given port. If not, add such a listen statement. if len(listen.split(":")) == 1: # Its listening to all interfaces if port not in listen_dirs and port_service not in listen_dirs: diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 9c25b1f17..804644975 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -65,8 +65,14 @@ Alias /.well-known/acme-challenge {0} """Make sure that we have the needed modules available for http01""" if self.configurator.conf("handle-modules"): - if "alias_module" not in self.configurator.parser.modules: - self.configurator.enable_mod("alias", temp=True) + needed_modules = ["alias"] + if self.configurator.version < (2, 4): + needed_modules.append("authz_host") + else: + needed_modules.append("authz_core") + for mod in needed_modules: + if mod + "_module" not in self.configurator.parser.modules: + self.configurator.enable_mod(mod, temp=True) def _mod_config(self): self.configurator.parser.add_include( diff --git a/certbot-apache/certbot_apache/override_debian.py b/certbot-apache/certbot_apache/override_debian.py index 6e2e34ba9..02dffc3f7 100644 --- a/certbot-apache/certbot_apache/override_debian.py +++ b/certbot-apache/certbot_apache/override_debian.py @@ -140,5 +140,5 @@ class DebianConfigurator(configurator.ApacheConfigurator): "a2dismod are configured correctly for certbot.") self.reverter.register_undo_command( - temp, [self.conf("dismod"), mod_name]) + temp, [self.conf("dismod"), "-f", mod_name]) util.run_script([self.conf("enmod"), mod_name]) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index f52b9d0cc..4e2a5faff 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -40,7 +40,6 @@ class ApacheHttp01Test(util.ApacheTest): def setUp(self, *args, **kwargs): super(ApacheHttp01Test, self).setUp(*args, **kwargs) - self.maxDiff = None self.account_key = self.rsa512jwk self.achalls = [] @@ -53,21 +52,51 @@ class ApacheHttp01Test(util.ApacheTest): domain="example{0}.com".format(i), account_key=self.account_key)) + modules = ["alias", "authz_core", "authz_host"] + for mod in modules: + self.config.parser.modules.add("mod_{0}.c".format(mod)) + self.config.parser.modules.add(mod + "_module") + from certbot_apache.http_01 import ApacheHttp01 - self.config.parser.modules.add("mod_alias.c") - self.config.parser.modules.add("alias_module") self.http = ApacheHttp01(self.config) def test_empty_perform(self): self.assertFalse(self.http.perform()) @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") - def test_add_alias_module(self, mock_enmod): + def test_enable_modules_22(self, mock_enmod): + self.config.version = (2, 2) + self.config.parser.modules.remove("authz_host_module") + self.config.parser.modules.remove("mod_authz_host.c") + + enmod_calls = self.common_enable_modules_test(mock_enmod) + self.assertEqual(enmod_calls[0][0][0], "authz_host") + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") + def test_enable_modules_24(self, mock_enmod): + self.config.parser.modules.remove("authz_core_module") + self.config.parser.modules.remove("mod_authz_core.c") + + enmod_calls = self.common_enable_modules_test(mock_enmod) + self.assertEqual(enmod_calls[0][0][0], "authz_core") + + def common_enable_modules_test(self, mock_enmod): + """Tests enabling mod_alias and other modules.""" self.config.parser.modules.remove("alias_module") self.config.parser.modules.remove("mod_alias.c") + self.http.prepare_http01_modules() + self.assertTrue(mock_enmod.called) - self.assertEqual(mock_enmod.call_args[0][0], "alias") + calls = mock_enmod.call_args_list + other_calls = [] + for call in calls: + if "alias" != call[0][0]: + other_calls.append(call) + + # If these lists are equal, we never enabled mod_alias + self.assertNotEqual(calls, other_calls) + return other_calls def common_perform_test(self, achalls): """Tests perform with the given achalls.""" diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 8f5162629..1ba1e2c34 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -103,7 +103,7 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc apache_challenge_location=config_path, backup_dir=backups, config_dir=config_dir, - http01_port="80", + http01_port=80, temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), in_progress_dir=os.path.join(backups, "IN_PROGRESS"), work_dir=work_dir) From 28dad825afae8a079811114d72a53a9833155d6c Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 11 Jan 2018 20:44:40 +0200 Subject: [PATCH 08/98] Do not try to remove temp dir if it wasn't created --- certbot-apache/certbot_apache/http_01.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 804644975..87721d290 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -58,7 +58,8 @@ Alias /.well-known/acme-challenge {0} def cleanup(self): """Cleanup the challenge directory.""" - shutil.rmtree(self.challenge_dir, ignore_errors=True) + if self.challenge_dir: + shutil.rmtree(self.challenge_dir, ignore_errors=True) self.challenge_dir = None def prepare_http01_modules(self): From 2cb9d9e2aa332e4fd53aab32ad1d3946438a0dd3 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 11 Jan 2018 17:06:23 -0800 Subject: [PATCH 09/98] Implement HTTP-01 challenge for Nginx (#5414) * get http01 challenge working * support multiple challenge types in configurator.py * update existing nginx tests * lint * refactor NginxHttp01 and NginxTlsSni01 to both now inherit from NginxChallengePerformer * remove TODO * challenges_test tests with both tlssni01 and http01 * Make challenges.py more abstract to make lint happier * add pylint disables to the tests to make pylint happier about the inheritance and abstraction situation * no need to cover raise NotImplementedError() lines * python3 compatibility * test that http01 perform is called * only remove ssl from addresses during http01 * Initialize addrs_to_add * Change Nginx http01 to modify server block so the site doesn't stop serving while getting a cert * pass existing unit tests * rename sni --> http01 in unit tests * lint * fix configurator test * select an http block instead of https * properly test for port number * use domains that have matching addresses * remove debugger * remove access_log and error_log cruft that wasn't being executed * continue to return None from choose_redirect_vhost when create_if_no_match is False * add nginx integration test --- certbot-nginx/certbot_nginx/configurator.py | 35 ++++-- certbot-nginx/certbot_nginx/http_01.py | 106 ++++++++++++++++ certbot-nginx/certbot_nginx/parser.py | 4 +- .../certbot_nginx/tests/configurator_test.py | 14 ++- .../certbot_nginx/tests/http_01_test.py | 113 ++++++++++++++++++ certbot-nginx/certbot_nginx/tests/util.py | 1 + .../tests/boulder-integration.conf.sh | 6 +- certbot-nginx/tests/boulder-integration.sh | 19 ++- 8 files changed, 272 insertions(+), 26 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/http_01.py create mode 100644 certbot-nginx/certbot_nginx/tests/http_01_test.py diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 8af474c5e..a77cf2bc3 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -26,6 +26,7 @@ from certbot_nginx import constants from certbot_nginx import nginxparser from certbot_nginx import parser from certbot_nginx import tls_sni_01 +from certbot_nginx import http_01 logger = logging.getLogger(__name__) @@ -208,7 +209,8 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param bool create_if_no_match: If we should create a new vhost from default - when there is no match found + when there is no match found. If we can't choose a default, raise a + MisconfigurationError. :returns: ssl vhost associated with name :rtype: :class:`~certbot_nginx.obj.VirtualHost` @@ -366,7 +368,7 @@ class NginxConfigurator(common.Installer): return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhost(self, target_name, port): + def choose_redirect_vhost(self, target_name, port, create_if_no_match=False): """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -380,12 +382,19 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param str port: port number + :param bool create_if_no_match: If we should create a new vhost from default + when there is no match found. If we can't choose a default, raise a + MisconfigurationError. + :returns: vhost associated with name :rtype: :class:`~certbot_nginx.obj.VirtualHost` """ matches = self._get_redirect_ranked_matches(target_name, port) - return self._select_best_name_match(matches) + vhost = self._select_best_name_match(matches) + if not vhost and create_if_no_match: + vhost = self._vhost_from_duplicated_default(target_name) + return vhost def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -394,7 +403,7 @@ class NginxConfigurator(common.Installer): Rank by how well these match target_name. :param str target_name: The name to match - :param str port: port number + :param str port: port number as a string :returns: list of dicts containing the vhost, the matching name, and the numerical rank :rtype: list @@ -840,7 +849,7 @@ class NginxConfigurator(common.Installer): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.TLSSNI01] + return [challenges.TLSSNI01, challenges.HTTP01] # Entry point in main.py for performing challenges def perform(self, achalls): @@ -853,15 +862,20 @@ class NginxConfigurator(common.Installer): """ self._chall_out += len(achalls) responses = [None] * len(achalls) - chall_doer = tls_sni_01.NginxTlsSni01(self) + sni_doer = tls_sni_01.NginxTlsSni01(self) + http_doer = http_01.NginxHttp01(self) for i, achall in enumerate(achalls): # Currently also have chall_doer hold associated index of the # challenge. This helps to put all of the responses back together # when they are all complete. - chall_doer.add_chall(achall, i) + if isinstance(achall.chall, challenges.HTTP01): + http_doer.add_chall(achall, i) + else: # tls-sni-01 + sni_doer.add_chall(achall, i) - sni_response = chall_doer.perform() + sni_response = sni_doer.perform() + http_response = http_doer.perform() # Must restart in order to activate the challenges. # Handled here because we may be able to load up other challenge types self.restart() @@ -869,8 +883,9 @@ class NginxConfigurator(common.Installer): # Go through all of the challenges and assign them to the proper place # in the responses return value. All responses must be in the same order # as the original challenges. - for i, resp in enumerate(sni_response): - responses[chall_doer.indices[i]] = resp + for chall_response, chall_doer in ((sni_response, sni_doer), (http_response, http_doer)): + for i, resp in enumerate(chall_response): + responses[chall_doer.indices[i]] = resp return responses diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py new file mode 100644 index 000000000..1f1e37891 --- /dev/null +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -0,0 +1,106 @@ +"""A class that performs HTTP-01 challenges for Nginx""" + +import logging +import os + +from acme import challenges + +from certbot.plugins import common + + +logger = logging.getLogger(__name__) + + +class NginxHttp01(common.ChallengePerformer): + """HTTP-01 authenticator for Nginx + + :ivar configurator: NginxConfigurator object + :type configurator: :class:`~nginx.configurator.NginxConfigurator` + + :ivar list achalls: Annotated + class:`~certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + challenges + + :param list indices: Meant to hold indices of challenges in a + larger array. NginxHttp01 is capable of solving many challenges + at once which causes an indexing issue within NginxConfigurator + who must return all responses in order. Imagine NginxConfigurator + maintaining state about where all of the http-01 Challenges, + TLS-SNI-01 Challenges belong in the response array. This is an + optional utility. + + """ + + def perform(self): + """Perform a challenge on Nginx. + + :returns: list of :class:`certbot.acme.challenges.HTTP01Response` + :rtype: list + + """ + if not self.achalls: + return [] + + responses = [x.response(x.account_key) for x in self.achalls] + + # Set up the configuration + self._mod_config() + + # Save reversible changes + self.configurator.save("HTTP Challenge", True) + + return responses + + def _add_bucket_directive(self): + """Modifies Nginx config to include server_names_hash_bucket_size directive.""" + root = self.configurator.parser.config_root + + bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128'] + + main = self.configurator.parser.parsed[root] + for line in main: + if line[0] == ['http']: + body = line[1] + found_bucket = False + posn = 0 + for inner_line in body: + if inner_line[0] == bucket_directive[1]: + if int(inner_line[1]) < int(bucket_directive[3]): + body[posn] = bucket_directive + found_bucket = True + posn += 1 + if not found_bucket: + body.insert(0, bucket_directive) + break + + def _mod_config(self): + """Modifies Nginx config to handle challenges. + + """ + self._add_bucket_directive() + + for achall in self.achalls: + self._mod_server_block(achall) + + def _get_validation_path(self, achall): + return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token")) + + def _mod_server_block(self, achall): + """Modifies a server block to respond to a challenge. + + :param achall: Annotated HTTP-01 challenge + :type achall: + :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + + """ + vhost = self.configurator.choose_redirect_vhost(achall.domain, + '%i' % self.configurator.config.http01_port, create_if_no_match=True) + validation = achall.validation(achall.account_key) + validation_path = self._get_validation_path(achall) + + location_directive = [[['location', ' ', '=', ' ', validation_path], + [['default_type', ' ', 'text/plain'], + ['return', ' ', '200', ' ', validation]]]] + + self.configurator.parser.add_server_directives(vhost, + location_directive, replace=False) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 9f13bc59f..5497f7e63 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -524,7 +524,7 @@ def _is_ssl_on_directive(entry): def _add_directives(directives, replace, block): """Adds or replaces directives in a config block. - When replace=False, it's an error to try and add a directive that already + When replace=False, it's an error to try and add a nonrepeatable directive that already exists in the config block with a conflicting value. When replace=True and a directive with the same name already exists in the @@ -545,7 +545,7 @@ def _add_directives(directives, replace, block): INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE]) +REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location']) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index e708b159a..7475df40c 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -100,7 +100,7 @@ class NginxConfiguratorTest(util.NginxTest): errors.PluginError, self.config.enhance, 'myhost', 'unknown_enhancement') def test_get_chall_pref(self): - self.assertEqual([challenges.TLSSNI01], + self.assertEqual([challenges.TLSSNI01, challenges.HTTP01], self.config.get_chall_pref('myhost')) def test_save(self): @@ -291,9 +291,11 @@ class NginxConfiguratorTest(util.NginxTest): parsed_migration_conf[0]) @mock.patch("certbot_nginx.configurator.tls_sni_01.NginxTlsSni01.perform") + @mock.patch("certbot_nginx.configurator.http_01.NginxHttp01.perform") @mock.patch("certbot_nginx.configurator.NginxConfigurator.restart") @mock.patch("certbot_nginx.configurator.NginxConfigurator.revert_challenge_config") - def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_perform): + def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_http_perform, + mock_tls_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( @@ -304,7 +306,7 @@ class NginxConfiguratorTest(util.NginxTest): ), domain="localhost", account_key=self.rsa512jwk) achall2 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=messages.ChallengeBody( - chall=challenges.TLSSNI01(token=b"m8TdO1qik4JVFtgPPurJmg"), + chall=challenges.HTTP01(token=b"m8TdO1qik4JVFtgPPurJmg"), uri="https://ca.org/chall1_uri", status=messages.Status("pending"), ), domain="example.com", account_key=self.rsa512jwk) @@ -314,10 +316,12 @@ class NginxConfiguratorTest(util.NginxTest): achall2.response(self.rsa512jwk), ] - mock_perform.return_value = expected + mock_tls_perform.return_value = expected[:1] + mock_http_perform.return_value = expected[1:] responses = self.config.perform([achall1, achall2]) - self.assertEqual(mock_perform.call_count, 1) + self.assertEqual(mock_tls_perform.call_count, 1) + self.assertEqual(mock_http_perform.call_count, 1) self.assertEqual(responses, expected) self.config.cleanup([achall1, achall2]) diff --git a/certbot-nginx/certbot_nginx/tests/http_01_test.py b/certbot-nginx/certbot_nginx/tests/http_01_test.py new file mode 100644 index 000000000..0f764e92e --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/http_01_test.py @@ -0,0 +1,113 @@ +"""Tests for certbot_nginx.http_01""" +import unittest +import shutil + +import mock +import six + +from acme import challenges + +from certbot import achallenges + +from certbot.plugins import common_test +from certbot.tests import acme_util + +from certbot_nginx.tests import util + + +class HttpPerformTest(util.NginxTest): + """Test the NginxHttp01 challenge.""" + + account_key = common_test.AUTH_KEY + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), "pending"), + domain="www.example.com", account_key=account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01( + token=b"\xba\xa9\xda? Date: Mon, 15 Jan 2018 01:22:22 +0200 Subject: [PATCH 10/98] Use static directory under workdir for HTTP challenges (#5428) * Use static directory under workdir for HTTP challenges * Handle the reverter file registration before opening file handle --- certbot-apache/certbot_apache/configurator.py | 1 - certbot-apache/certbot_apache/http_01.py | 19 ++++++++----------- .../certbot_apache/tests/configurator_test.py | 5 ----- .../certbot_apache/tests/http_01_test.py | 6 +++--- 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 8d6995211..b87189dc0 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1956,7 +1956,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.revert_challenge_config() self.restart() self.parser.reset_modules() - self.http_doer.cleanup() def install_ssl_options_conf(self, options_ssl, options_ssl_digest): """Copy Certbot's SSL options file into the system's config dir if required.""" diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 87721d290..3b942823d 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -1,8 +1,6 @@ """A class that performs HTTP-01 challenges for Apache""" import logging import os -import shutil -import tempfile from certbot.plugins import common @@ -35,7 +33,9 @@ Alias /.well-known/acme-challenge {0} self.challenge_conf = os.path.join( self.configurator.conf("challenge-location"), "le_http_01_challenge.conf") - self.challenge_dir = None + self.challenge_dir = os.path.join( + self.configurator.config.work_dir, + "http_challenges") def perform(self): """Perform all HTTP-01 challenges.""" @@ -56,12 +56,6 @@ Alias /.well-known/acme-challenge {0} return responses - def cleanup(self): - """Cleanup the challenge directory.""" - if self.challenge_dir: - shutil.rmtree(self.challenge_dir, ignore_errors=True) - self.challenge_dir = None - def prepare_http01_modules(self): """Make sure that we have the needed modules available for http01""" @@ -92,8 +86,9 @@ Alias /.well-known/acme-challenge {0} new_conf.write(config_text) def _set_up_challenges(self): - self.challenge_dir = tempfile.mkdtemp() - os.chmod(self.challenge_dir, 0o755) + if not os.path.isdir(self.challenge_dir): + os.makedirs(self.challenge_dir) + os.chmod(self.challenge_dir, 0o755) responses = [] for achall in self.achalls: @@ -105,6 +100,8 @@ Alias /.well-known/acme-challenge {0} response, validation = achall.response_and_validation() name = os.path.join(self.challenge_dir, achall.chall.encode("token")) + + self.configurator.reverter.register_file_creation(True, name) with open(name, 'wb') as f: f.write(validation.encode()) os.chmod(name, 0o644) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index ea3867061..df67104da 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -747,7 +747,6 @@ class MultipleVhostsTest(util.ApacheTest): def test_cleanup(self, mock_cfg, mock_restart): mock_cfg.return_value = "" _, achalls = self.get_key_and_achalls() - self.config.http_doer = mock.MagicMock() for achall in achalls: self.config._chall_out.add(achall) # pylint: disable=protected-access @@ -756,10 +755,8 @@ class MultipleVhostsTest(util.ApacheTest): self.config.cleanup([achall]) if i == len(achalls) - 1: self.assertTrue(mock_restart.called) - self.assertTrue(self.config.http_doer.cleanup.called) else: self.assertFalse(mock_restart.called) - self.assertFalse(self.config.http_doer.cleanup.called) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") @@ -773,11 +770,9 @@ class MultipleVhostsTest(util.ApacheTest): self.config.cleanup([achalls[-1]]) self.assertFalse(mock_restart.called) - self.assertFalse(self.config.http_doer.cleanup.called) self.config.cleanup(achalls) self.assertTrue(mock_restart.called) - self.assertTrue(self.config.http_doer.cleanup.called) @mock.patch("certbot.util.run_script") def test_get_version(self, mock_script): diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 4e2a5faff..204d9a76c 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -100,6 +100,8 @@ class ApacheHttp01Test(util.ApacheTest): def common_perform_test(self, achalls): """Tests perform with the given achalls.""" + challenge_dir = self.http.challenge_dir + self.assertFalse(os.path.exists(challenge_dir)) for achall in achalls: self.http.add_chall(achall) @@ -114,9 +116,7 @@ class ApacheHttp01Test(util.ApacheTest): for achall in achalls: self._test_challenge_file(achall) - challenge_dir = self.http.challenge_dir - self.http.cleanup() - self.assertFalse(os.path.exists(challenge_dir)) + self.assertTrue(os.path.exists(challenge_dir)) def _test_challenge_conf(self): self.assertEqual( From 368ca0c1096aa198b2e841dd52778646812e8141 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 15 Jan 2018 20:47:03 -0800 Subject: [PATCH 11/98] Small cleanup for Apache HTTP-01 * Remove http_doer from self * Refactor _find_best_vhost --- certbot-apache/certbot_apache/configurator.py | 29 +++++++++++-------- .../certbot_apache/tests/configurator_test.py | 3 +- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index b87189dc0..05c838123 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -164,9 +164,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "ensure-http-header": self._set_http_header, "staple-ocsp": self._enable_ocsp_stapling} - # This will be set during the perform function - self.http_doer = None - @property def mod_ssl_conf(self): """Full absolute path to SSL configuration file.""" @@ -439,12 +436,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return True return False - def _find_best_vhost(self, target_name): + def _find_best_vhost(self, target_name, vhosts=None): """Finds the best vhost for a target_name. This does not upgrade a vhost to HTTPS... it only finds the most appropriate vhost for the given target_name. + :param str target_name: domain handled by the desired vhost + :param vhosts: vhosts to consider + :type vhosts: `collections.Iterable` of :class:`~certbot_apache.obj.VirtualHost` + :returns: VHost or None """ @@ -456,7 +457,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Points 1 - Address name with no SSL best_candidate = None best_points = 0 - for vhost in self.vhosts: + + if vhosts is None: + vhosts = self.vhosts + + for vhost in vhosts: if vhost.modmacro is True: continue names = vhost.get_names() @@ -481,7 +486,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # No winners here... is there only one reasonable vhost? if best_candidate is None: # reasonable == Not all _default_ addrs - vhosts = self._non_default_vhosts() + vhosts = self._non_default_vhosts(vhosts) # remove mod_macro hosts from reasonable vhosts reasonable_vhosts = [vh for vh in vhosts if vh.modmacro is False] @@ -490,9 +495,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return best_candidate - def _non_default_vhosts(self): + def _non_default_vhosts(self, vhosts): """Return all non _default_ only vhosts.""" - return [vh for vh in self.vhosts if not all( + return [vh for vh in vhosts if not all( addr.get_addr() == "_default_" for addr in vh.addrs )] @@ -1911,7 +1916,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ self._chall_out.update(achalls) responses = [None] * len(achalls) - self.http_doer = http_01.ApacheHttp01(self) + http_doer = http_01.ApacheHttp01(self) sni_doer = tls_sni_01.ApacheTlsSni01(self) for i, achall in enumerate(achalls): @@ -1919,11 +1924,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # challenge. This helps to put all of the responses back together # when they are all complete. if isinstance(achall.chall, challenges.HTTP01): - self.http_doer.add_chall(achall, i) + http_doer.add_chall(achall, i) else: # tls-sni-01 sni_doer.add_chall(achall, i) - http_response = self.http_doer.perform() + http_response = http_doer.perform() sni_response = sni_doer.perform() if http_response or sni_response: # Must reload in order to activate the challenges. @@ -1935,7 +1940,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # of identifying when the new configuration is being used. time.sleep(3) - self._update_responses(responses, http_response, self.http_doer) + self._update_responses(responses, http_response, http_doer) self._update_responses(responses, sni_response, sni_doer) return responses diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index df67104da..c846d2d42 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -305,7 +305,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_non_default_vhosts(self): # pylint: disable=protected-access - self.assertEqual(len(self.config._non_default_vhosts()), 8) + vhosts = self.config._non_default_vhosts(self.config.vhosts) + self.assertEqual(len(vhosts), 8) def test_deploy_cert_enable_new_vhost(self): # Create From 7e463bccad469c1fa7cda6aba09b1e459a514e2d Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 16 Jan 2018 14:58:45 -0800 Subject: [PATCH 12/98] Handle more edge cases for HTTP-01 support in Nginx (#5421) * only when using http01, only match default_server by port * import errors * put back in the code that creates a dummy block, but only when we can't find anything else --- certbot-nginx/certbot_nginx/configurator.py | 30 +++--- certbot-nginx/certbot_nginx/http_01.py | 107 ++++++++++++++++++-- 2 files changed, 113 insertions(+), 24 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index a77cf2bc3..bb2933a39 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -261,9 +261,9 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain): + def _vhost_from_duplicated_default(self, domain, port=None): if self.new_vhost is None: - default_vhost = self._get_default_vhost() + default_vhost = self._get_default_vhost(port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, delete_default=True) self.new_vhost.names = set() @@ -278,15 +278,16 @@ class NginxConfigurator(common.Installer): name_block[0].append(name) self.parser.add_server_directives(vhost, name_block, replace=True) - def _get_default_vhost(self): + def _get_default_vhost(self, port): vhost_list = self.parser.get_vhosts() # if one has default_server set, return that one default_vhosts = [] for vhost in vhost_list: for addr in vhost.addrs: if addr.default: - default_vhosts.append(vhost) - break + if port is None or self._port_matches(port, addr.get_port()): + default_vhosts.append(vhost) + break if len(default_vhosts) == 1: return default_vhosts[0] @@ -393,9 +394,17 @@ class NginxConfigurator(common.Installer): matches = self._get_redirect_ranked_matches(target_name, port) vhost = self._select_best_name_match(matches) if not vhost and create_if_no_match: - vhost = self._vhost_from_duplicated_default(target_name) + vhost = self._vhost_from_duplicated_default(target_name, port=port) return vhost + def _port_matches(self, test_port, matching_port): + # test_port is a number, matching is a number or "" or None + if matching_port == "" or matching_port is None: + # if no port is specified, Nginx defaults to listening on port 80. + return test_port == self.DEFAULT_LISTEN_PORT + else: + return test_port == matching_port + def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -410,13 +419,6 @@ class NginxConfigurator(common.Installer): """ all_vhosts = self.parser.get_vhosts() - def _port_matches(test_port, matching_port): - # test_port is a number, matching is a number or "" or None - if matching_port == "" or matching_port is None: - # if no port is specified, Nginx defaults to listening on port 80. - return test_port == self.DEFAULT_LISTEN_PORT - else: - return test_port == matching_port def _vhost_matches(vhost, port): found_matching_port = False @@ -426,7 +428,7 @@ class NginxConfigurator(common.Installer): found_matching_port = (port == self.DEFAULT_LISTEN_PORT) else: for addr in vhost.addrs: - if _port_matches(port, addr.get_port()) and addr.ssl == False: + if self._port_matches(port, addr.get_port()) and addr.ssl == False: found_matching_port = True if found_matching_port: diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 1f1e37891..c25081ae0 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -5,8 +5,12 @@ import os from acme import challenges +from certbot import errors from certbot.plugins import common +from certbot_nginx import obj +from certbot_nginx import nginxparser + logger = logging.getLogger(__name__) @@ -31,6 +35,13 @@ class NginxHttp01(common.ChallengePerformer): """ + def __init__(self, configurator): + super(NginxHttp01, self).__init__(configurator) + self.challenge_conf = os.path.join( + configurator.config.config_dir, "le_http_01_cert_challenge.conf") + self._ipv6 = None + self._ipv6only = None + def perform(self): """Perform a challenge on Nginx. @@ -51,8 +62,16 @@ class NginxHttp01(common.ChallengePerformer): return responses - def _add_bucket_directive(self): - """Modifies Nginx config to include server_names_hash_bucket_size directive.""" + def _mod_config(self): + """Modifies Nginx config to include server_names_hash_bucket_size directive + and server challenge blocks. + + :raises .MisconfigurationError: + Unable to find a suitable HTTP block in which to include + authenticator hosts. + """ + included = False + include_directive = ['\n', 'include', ' ', self.challenge_conf] root = self.configurator.parser.config_root bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128'] @@ -71,21 +90,82 @@ class NginxHttp01(common.ChallengePerformer): posn += 1 if not found_bucket: body.insert(0, bucket_directive) + if include_directive not in body: + body.insert(0, include_directive) + included = True break + if not included: + raise errors.MisconfigurationError( + 'Certbot could not find a block to include ' + 'challenges in %s.' % root) + config = [self._make_or_mod_server_block(achall) for achall in self.achalls] + config = [x for x in config if x is not None] + config = nginxparser.UnspacedList(config) - def _mod_config(self): - """Modifies Nginx config to handle challenges. + self.configurator.reverter.register_file_creation( + True, self.challenge_conf) + with open(self.challenge_conf, "w") as new_conf: + nginxparser.dump(config, new_conf) + + def _default_listen_addresses(self): + """Finds addresses for a challenge block to listen on. + :returns: list of :class:`certbot_nginx.obj.Addr` to apply + :rtype: list """ - self._add_bucket_directive() + addresses = [] + default_addr = "%s" % self.configurator.config.http01_port + ipv6_addr = "[::]:{0}".format( + self.configurator.config.http01_port) + port = self.configurator.config.http01_port - for achall in self.achalls: - self._mod_server_block(achall) + if self._ipv6 is None or self._ipv6only is None: + self._ipv6, self._ipv6only = self.configurator.ipv6_info(port) + ipv6, ipv6only = self._ipv6, self._ipv6only + + if ipv6: + # If IPv6 is active in Nginx configuration + if not ipv6only: + # If ipv6only=on is not already present in the config + ipv6_addr = ipv6_addr + " ipv6only=on" + addresses = [obj.Addr.fromstring(default_addr), + obj.Addr.fromstring(ipv6_addr)] + logger.info(("Using default addresses %s and %s for authentication."), + default_addr, + ipv6_addr) + else: + addresses = [obj.Addr.fromstring(default_addr)] + logger.info("Using default address %s for authentication.", + default_addr) + return addresses def _get_validation_path(self, achall): return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token")) - def _mod_server_block(self, achall): + def _make_server_block(self, achall): + """Creates a server block for a challenge. + :param achall: Annotated HTTP-01 challenge + :type achall: + :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :param list addrs: addresses of challenged domain + :class:`list` of type :class:`~nginx.obj.Addr` + :returns: server block for the challenge host + :rtype: list + """ + addrs = self._default_listen_addresses() + block = [['listen', ' ', addr.to_string(include_default=False)] for addr in addrs] + + validation = achall.validation(achall.account_key) + validation_path = self._get_validation_path(achall) + + block.extend([['server_name', ' ', achall.domain], + [['location', ' ', '=', ' ', validation_path], + [['default_type', ' ', 'text/plain'], + ['return', ' ', '200', ' ', validation]]]]) + # TODO: do we want to return something else if they otherwise access this block? + return [['server'], block] + + def _make_or_mod_server_block(self, achall): """Modifies a server block to respond to a challenge. :param achall: Annotated HTTP-01 challenge @@ -93,8 +173,15 @@ class NginxHttp01(common.ChallengePerformer): :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` """ - vhost = self.configurator.choose_redirect_vhost(achall.domain, - '%i' % self.configurator.config.http01_port, create_if_no_match=True) + try: + vhost = self.configurator.choose_redirect_vhost(achall.domain, + '%i' % self.configurator.config.http01_port, create_if_no_match=True) + except errors.MisconfigurationError: + # Couldn't find either a matching name+port server block + # or a port+default_server block, so create a dummy block + return self._make_server_block(achall) + + # Modify existing server block validation = achall.validation(achall.account_key) validation_path = self._get_validation_path(achall) From 314c5f19e51ccefc740f87751c589e2ca0fe9754 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 16 Jan 2018 20:33:25 +0200 Subject: [PATCH 13/98] Set up vhost discovery and overrides for HTTP-01 * Finalized HTTP vhost discovery and added overrides * Include overrides to every VirtualHost --- certbot-apache/certbot_apache/configurator.py | 14 ++++- certbot-apache/certbot_apache/http_01.py | 60 +++++++++++++------ certbot-apache/certbot_apache/parser.py | 17 ++++++ .../certbot_apache/tests/http_01_test.py | 6 +- .../certbot_apache/tests/parser_test.py | 17 ++++++ 5 files changed, 92 insertions(+), 22 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 05c838123..8f1aff8d7 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -436,6 +436,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return True return False + def find_best_http_vhost(self, target): + """Returns non-HTTPS vhost objects found from the Apache config + + :param str target: Domain name of the desired VirtualHost + + :returns: VirtualHost object that's the best match for target name + :rtype: `obj.VirtualHost` or None + """ + nonssl_vhosts = [i for i in self.vhosts if not i.ssl] + return self._find_best_vhost(target, nonssl_vhosts) + + def _find_best_vhost(self, target_name, vhosts=None): """Finds the best vhost for a target_name. @@ -508,7 +520,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): virtual host addresses :rtype: set - """ + """ all_names = set() vhost_macro = [] diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 3b942823d..a2cd43845 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -7,26 +7,38 @@ from certbot.plugins import common logger = logging.getLogger(__name__) class ApacheHttp01(common.TLSSNI01): - """Class that performs HTPP-01 challenges within the Apache configurator.""" + """Class that performs HTTP-01 challenges within the Apache configurator.""" - CONFIG_TEMPLATE24 = """\ -Alias /.well-known/acme-challenge {0} + CONFIG_TEMPLATE_COMMON = """\ + Alias /.well-known/acme-challenge {0}" - - Require all granted - - -""" + + ProxyPass "/.well-known/acme-challenge" ! + + """ CONFIG_TEMPLATE22 = """\ -Alias /.well-known/acme-challenge {0} + + RewriteEngine on + RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [L,S=9999] + - - Order allow,deny - Allow from all - + + Order allow deny + Allow from all + + """ -""" + CONFIG_TEMPLATE24 = """\ + + RewriteEngine on + RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [END] + + + + Require all granted + + """ def __init__(self, *args, **kwargs): super(ApacheHttp01, self).__init__(*args, **kwargs) @@ -50,6 +62,7 @@ Alias /.well-known/acme-challenge {0} self.prepare_http01_modules() responses = self._set_up_challenges() + self._mod_config() # Save reversible changes self.configurator.save("HTTP Challenge", True) @@ -70,21 +83,26 @@ Alias /.well-known/acme-challenge {0} self.configurator.enable_mod(mod, temp=True) def _mod_config(self): - self.configurator.parser.add_include( - self.configurator.parser.loc["default"], self.challenge_conf) + for chall in self.achalls: + vh = self.configurator.find_best_http_vhost(chall.domain) + if vh: + self._set_up_include_directive(vh) + self.configurator.reverter.register_file_creation( True, self.challenge_conf) if self.configurator.version < (2, 4): - config_template = self.CONFIG_TEMPLATE22 + config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE22 else: - config_template = self.CONFIG_TEMPLATE24 + config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE24 + config_text = config_template.format(self.challenge_dir) logger.debug("writing a config file with text:\n %s", config_text) with open(self.challenge_conf, "w") as new_conf: new_conf.write(config_text) + def _set_up_challenges(self): if not os.path.isdir(self.challenge_dir): os.makedirs(self.challenge_dir) @@ -107,3 +125,9 @@ Alias /.well-known/acme-challenge {0} os.chmod(name, 0o644) return response + + def _set_up_include_directive(self, vhost): + """Includes override configuration to the beginning of VirtualHost. + Note that this include isn't added to Augeas search tree""" + self.configurator.parser.add_dir_beginning(vhost.path, "Include", + self.challenge_conf) diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 7715d2c35..d7da1e55e 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -332,6 +332,23 @@ class ApacheParser(object): else: self.aug.set(aug_conf_path + "/directive[last()]/arg", args) + def add_dir_beginning(self, aug_conf_path, dirname, args): + """Adds the directive to the beginning of defined aug_conf_path. + + :param str aug_conf_path: Augeas configuration path to add directive + :param str dirname: Directive to add + :param args: Value of the directive. ie. Listen 443, 443 is arg + :type args: list or str + """ + first_dir = aug_conf_path + "/directive[1]" + self.aug.insert(first_dir, "directive", True) + self.aug.set(first_dir, dirname) + if isinstance(args, list): + for i, value in enumerate(args, 1): + self.aug.set(first_dir + "/arg[%d]" % (i), value) + else: + self.aug.set(first_dir + "/arg", args) + def find_dir(self, directive, arg=None, start=None, exclude=True): """Finds directive in the configuration. diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 204d9a76c..54b4ff208 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -119,9 +119,9 @@ class ApacheHttp01Test(util.ApacheTest): self.assertTrue(os.path.exists(challenge_dir)) def _test_challenge_conf(self): - self.assertEqual( - len(self.config.parser.find_dir( - "Include", self.http.challenge_conf)), 1) + #self.assertEqual( + # len(self.config.parser.find_dir( + # "Include", self.http.challenge_conf)), 1) with open(self.http.challenge_conf) as f: conf_contents = f.read() diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index a9eb129c2..4496781c9 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -66,6 +66,23 @@ class BasicParserTest(util.ParserTest): for i, match in enumerate(matches): self.assertEqual(self.parser.aug.get(match), str(i + 1)) + def test_add_dir_beginning(self): + aug_default = "/files" + self.parser.loc["default"] + self.parser.add_dir_beginning(aug_default, + "AddDirectiveBeginning", + "testBegin") + + self.assertTrue( + self.parser.find_dir("AddDirectiveBeginning", "testBegin", aug_default)) + + self.assertEqual( + self.parser.aug.get(aug_default+"/directive[1]"), + "AddDirectiveBeginning") + self.parser.add_dir_beginning(aug_default, "AddList", ["1", "2", "3", "4"]) + matches = self.parser.find_dir("AddList", None, aug_default) + for i, match in enumerate(matches): + self.assertEqual(self.parser.aug.get(match), str(i + 1)) + def test_empty_arg(self): self.assertEquals(None, self.parser.get_arg("/files/whatever/nonexistent")) From f420b19492075b569eea349d5c6a4ec9bafa236f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 Jan 2018 18:16:33 -0800 Subject: [PATCH 14/98] Apache HTTP01 Improvements * Fix docstring quote spacing * Remove unneeded directives * Enable mod_rewrite * Remove ifmod rewrite * Use stricter rewriterule * Uncomment tests * Fix order args * Remove S which doesn't seem to work across contexts * Use double backslash to make pylint * Fix enmod test * Fix http-01 tests * Test for rewrite * check for Include in vhost * add test_same_vhost * Don't add includes twice * Include default vhosts in search * Respect port in find_best_http_vhost * Add find_best_http_vhost port test * Filter by port in http01 --- certbot-apache/certbot_apache/configurator.py | 23 ++++--- certbot-apache/certbot_apache/http_01.py | 36 ++++------ .../certbot_apache/tests/configurator_test.py | 18 ++++- .../certbot_apache/tests/http_01_test.py | 67 ++++++++++++++----- .../apache2/sites-available/certbot.conf | 1 + certbot-apache/certbot_apache/tests/util.py | 2 +- 6 files changed, 97 insertions(+), 50 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 8f1aff8d7..b32eda921 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -436,19 +436,24 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return True return False - def find_best_http_vhost(self, target): + def find_best_http_vhost(self, target, filter_defaults, port="80"): """Returns non-HTTPS vhost objects found from the Apache config :param str target: Domain name of the desired VirtualHost + :param bool filter_defaults: whether _default_ vhosts should be + included if it is the best match + :param str port: port number the vhost should be listening on :returns: VirtualHost object that's the best match for target name :rtype: `obj.VirtualHost` or None """ - nonssl_vhosts = [i for i in self.vhosts if not i.ssl] - return self._find_best_vhost(target, nonssl_vhosts) + filtered_vhosts = [] + for vhost in self.vhosts: + if any(a.is_wildcard() or a.get_port() == port for a in vhost.addrs) and not vhost.ssl: + filtered_vhosts.append(vhost) + return self._find_best_vhost(target, filtered_vhosts, filter_defaults) - - def _find_best_vhost(self, target_name, vhosts=None): + def _find_best_vhost(self, target_name, vhosts=None, filter_defaults=True): """Finds the best vhost for a target_name. This does not upgrade a vhost to HTTPS... it only finds the most @@ -457,6 +462,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str target_name: domain handled by the desired vhost :param vhosts: vhosts to consider :type vhosts: `collections.Iterable` of :class:`~certbot_apache.obj.VirtualHost` + :param bool filter_defaults: whether a vhost with a _default_ + addr is acceptable :returns: VHost or None @@ -497,8 +504,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # No winners here... is there only one reasonable vhost? if best_candidate is None: - # reasonable == Not all _default_ addrs - vhosts = self._non_default_vhosts(vhosts) + if filter_defaults: + vhosts = self._non_default_vhosts(vhosts) # remove mod_macro hosts from reasonable vhosts reasonable_vhosts = [vh for vh in vhosts if vh.modmacro is False] @@ -520,7 +527,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): virtual host addresses :rtype: set - """ + """ all_names = set() vhost_macro = [] diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index a2cd43845..edcca8c9c 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -9,31 +9,19 @@ logger = logging.getLogger(__name__) class ApacheHttp01(common.TLSSNI01): """Class that performs HTTP-01 challenges within the Apache configurator.""" - CONFIG_TEMPLATE_COMMON = """\ - Alias /.well-known/acme-challenge {0}" - - - ProxyPass "/.well-known/acme-challenge" ! - - """ - CONFIG_TEMPLATE22 = """\ - - RewriteEngine on - RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [L,S=9999] - + RewriteEngine on + RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L] - Order allow deny + Order Allow,Deny Allow from all """ CONFIG_TEMPLATE24 = """\ - - RewriteEngine on - RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [END] - + RewriteEngine on + RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END] Require all granted @@ -73,7 +61,7 @@ class ApacheHttp01(common.TLSSNI01): """Make sure that we have the needed modules available for http01""" if self.configurator.conf("handle-modules"): - needed_modules = ["alias"] + needed_modules = ["rewrite"] if self.configurator.version < (2, 4): needed_modules.append("authz_host") else: @@ -83,18 +71,22 @@ class ApacheHttp01(common.TLSSNI01): self.configurator.enable_mod(mod, temp=True) def _mod_config(self): + moded_vhosts = set() for chall in self.achalls: - vh = self.configurator.find_best_http_vhost(chall.domain) - if vh: + vh = self.configurator.find_best_http_vhost( + chall.domain, filter_defaults=False, + port=str(self.configurator.config.http01_port)) + if vh and vh not in moded_vhosts: self._set_up_include_directive(vh) + moded_vhosts.add(vh) self.configurator.reverter.register_file_creation( True, self.challenge_conf) if self.configurator.version < (2, 4): - config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE22 + config_template = self.CONFIG_TEMPLATE22 else: - config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE24 + config_template = self.CONFIG_TEMPLATE24 config_text = config_template.format(self.challenge_dir) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index c846d2d42..530d75a92 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -126,7 +126,7 @@ class MultipleVhostsTest(util.ApacheTest): names = self.config.get_all_names() self.assertEqual(names, set( ["certbot.demo", "ocspvhost.com", "encryption-example.demo", - "nonsym.link", "vhost.in.rootconf"] + "nonsym.link", "vhost.in.rootconf", "www.certbot.demo"] )) @certbot_util.patch_get_utility() @@ -146,7 +146,7 @@ class MultipleVhostsTest(util.ApacheTest): names = self.config.get_all_names() # Names get filtered, only 5 are returned - self.assertEqual(len(names), 7) + self.assertEqual(len(names), 8) self.assertTrue("zombo.com" in names) self.assertTrue("google.com" in names) self.assertTrue("certbot.demo" in names) @@ -260,6 +260,20 @@ class MultipleVhostsTest(util.ApacheTest): self.assertRaises( errors.PluginError, self.config.choose_vhost, "none.com") + def test_find_best_http_vhost_default(self): + vh = obj.VirtualHost( + "fp", "ap", set([obj.Addr.fromstring("_default_:80")]), False, True) + self.config.vhosts = [vh] + self.assertEqual(self.config.find_best_http_vhost("foo.bar", False), vh) + + def test_find_best_http_vhost_port(self): + port = "8080" + vh = obj.VirtualHost( + "fp", "ap", set([obj.Addr.fromstring("*:" + port)]), + False, True, "encryption-example.demo") + self.config.vhosts.append(vh) + self.assertEqual(self.config.find_best_http_vhost("foo.bar", False, port), vh) + def test_findbest_continues_on_short_domain(self): # pylint: disable=protected-access chosen_vhost = self.config._find_best_vhost("purple.com") diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 54b4ff208..768d904e8 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -22,8 +22,9 @@ class ApacheHttp01TestMeta(type): def _gen_test(num_achalls, minor_version): def _test(self): achalls = self.achalls[:num_achalls] + vhosts = self.vhosts[:num_achalls] self.config.version = (2, minor_version) - self.common_perform_test(achalls) + self.common_perform_test(achalls, vhosts) return _test for i in range(1, NUM_ACHALLS + 1): @@ -43,16 +44,30 @@ class ApacheHttp01Test(util.ApacheTest): self.account_key = self.rsa512jwk self.achalls = [] + self.vhosts = [] + vhost_index = 0 for i in range(NUM_ACHALLS): + domain = None + # Find a vhost with a name/alias we can use + for j in range(vhost_index + 1, len(self.config.vhosts)): + vhost = self.config.vhosts[j] + domain = vhost.name if vhost.name else next(iter(vhost.aliases), None) + if domain: + self.vhosts.append(vhost) + vhost_index = j + 1 + break + else: # pragma: no cover + # If we didn't find a domain, we shouldn't continue the test. + self.fail("No usable vhost found") + self.achalls.append( achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( challenges.HTTP01(token=((chr(ord('a') + i) * 16))), "pending"), - domain="example{0}.com".format(i), - account_key=self.account_key)) + domain=domain, account_key=self.account_key)) - modules = ["alias", "authz_core", "authz_host"] + modules = ["rewrite", "authz_core", "authz_host"] for mod in modules: self.config.parser.modules.add("mod_{0}.c".format(mod)) self.config.parser.modules.add(mod + "_module") @@ -81,9 +96,9 @@ class ApacheHttp01Test(util.ApacheTest): self.assertEqual(enmod_calls[0][0][0], "authz_core") def common_enable_modules_test(self, mock_enmod): - """Tests enabling mod_alias and other modules.""" - self.config.parser.modules.remove("alias_module") - self.config.parser.modules.remove("mod_alias.c") + """Tests enabling mod_rewrite and other modules.""" + self.config.parser.modules.remove("rewrite_module") + self.config.parser.modules.remove("mod_rewrite.c") self.http.prepare_http01_modules() @@ -91,14 +106,30 @@ class ApacheHttp01Test(util.ApacheTest): calls = mock_enmod.call_args_list other_calls = [] for call in calls: - if "alias" != call[0][0]: + if "rewrite" != call[0][0]: other_calls.append(call) - # If these lists are equal, we never enabled mod_alias + # If these lists are equal, we never enabled mod_rewrite self.assertNotEqual(calls, other_calls) return other_calls - def common_perform_test(self, achalls): + def test_same_vhost(self): + vhost = next(v for v in self.config.vhosts if v.name == "certbot.demo") + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'a' * 16))), + "pending"), + domain=vhost.name, account_key=self.account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'b' * 16))), + "pending"), + domain=next(iter(vhost.aliases)), account_key=self.account_key) + ] + self.common_perform_test(achalls, [vhost]) + + def common_perform_test(self, achalls, vhosts): """Tests perform with the given achalls.""" challenge_dir = self.http.challenge_dir self.assertFalse(os.path.exists(challenge_dir)) @@ -116,19 +147,21 @@ class ApacheHttp01Test(util.ApacheTest): for achall in achalls: self._test_challenge_file(achall) + for vhost in vhosts: + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf, + vhost.path) + self.assertEqual(len(matches), 1) + self.assertTrue(os.path.exists(challenge_dir)) def _test_challenge_conf(self): - #self.assertEqual( - # len(self.config.parser.find_dir( - # "Include", self.http.challenge_conf)), 1) - with open(self.http.challenge_conf) as f: conf_contents = f.read() - alias_fmt = "Alias /.well-known/acme-challenge {0}" - alias = alias_fmt.format(self.http.challenge_dir) - self.assertTrue(alias in conf_contents) + self.assertTrue("RewriteEngine on" in conf_contents) + self.assertTrue("RewriteRule" in conf_contents) + self.assertTrue(self.http.challenge_dir in conf_contents) if self.config.version < (2, 4): self.assertTrue("Allow from all" in conf_contents) else: diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf index b3147a523..965ca2222 100644 --- a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf @@ -1,5 +1,6 @@ ServerName certbot.demo +ServerAlias www.certbot.demo ServerAdmin webmaster@localhost DocumentRoot /var/www-certbot-reworld/static/ diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 1ba1e2c34..1daaa00c5 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -170,7 +170,7 @@ def get_vh_truth(temp_dir, config_name): os.path.join(prefix, "certbot.conf"), os.path.join(aug_pre, "certbot.conf/VirtualHost"), set([obj.Addr.fromstring("*:80")]), False, True, - "certbot.demo"), + "certbot.demo", aliases=["www.certbot.demo"]), obj.VirtualHost( os.path.join(prefix, "mod_macro-example.conf"), os.path.join(aug_pre, From b8f288a3720fda4f2f1ef56ada30f887f5b41a63 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 17 Jan 2018 14:08:45 +0200 Subject: [PATCH 15/98] Add include to every VirtualHost if definite one not found based on name --- certbot-apache/certbot_apache/http_01.py | 19 ++++++++++++++----- .../certbot_apache/tests/http_01_test.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index edcca8c9c..2b2b8e796 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -36,6 +36,7 @@ class ApacheHttp01(common.TLSSNI01): self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") + self.moded_vhosts = set() def perform(self): """Perform all HTTP-01 challenges.""" @@ -71,14 +72,16 @@ class ApacheHttp01(common.TLSSNI01): self.configurator.enable_mod(mod, temp=True) def _mod_config(self): - moded_vhosts = set() for chall in self.achalls: vh = self.configurator.find_best_http_vhost( chall.domain, filter_defaults=False, port=str(self.configurator.config.http01_port)) - if vh and vh not in moded_vhosts: + if vh: self._set_up_include_directive(vh) - moded_vhosts.add(vh) + else: + for vh in self.configurator.vhosts: + if not vh.ssl: + self._set_up_include_directive(vh) self.configurator.reverter.register_file_creation( True, self.challenge_conf) @@ -121,5 +124,11 @@ class ApacheHttp01(common.TLSSNI01): def _set_up_include_directive(self, vhost): """Includes override configuration to the beginning of VirtualHost. Note that this include isn't added to Augeas search tree""" - self.configurator.parser.add_dir_beginning(vhost.path, "Include", - self.challenge_conf) + + if vhost not in self.moded_vhosts: + logger.debug( + "Adding a temporary challenge validation Include for name: %s " + + "in: %s", vhost.name, vhost.filep) + self.configurator.parser.add_dir_beginning( + vhost.path, "Include", self.challenge_conf) + self.moded_vhosts.add(vhost) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 768d904e8..12f571354 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -129,6 +129,16 @@ class ApacheHttp01Test(util.ApacheTest): ] self.common_perform_test(achalls, [vhost]) + def test_anonymous_vhost(self): + vhosts = [v for v in self.config.vhosts if not v.ssl] + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'a' * 16))), + "pending"), + domain="something.nonexistent", account_key=self.account_key)] + self.common_perform_test(achalls, vhosts) + def common_perform_test(self, achalls, vhosts): """Tests perform with the given achalls.""" challenge_dir = self.http.challenge_dir From 2c379cd363143720644ff38eccc85a349c2f2b00 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 17 Jan 2018 08:01:44 -0800 Subject: [PATCH 16/98] Add a rewrite directive for the .well-known location so we don't hit existing rewrites (#5436) --- certbot-nginx/certbot_nginx/http_01.py | 5 +++++ certbot-nginx/certbot_nginx/parser.py | 28 +++++++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index c25081ae0..2885e7ac0 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -191,3 +191,8 @@ class NginxHttp01(common.ChallengePerformer): self.configurator.parser.add_server_directives(vhost, location_directive, replace=False) + + rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', + ' ', '$1', ' ', 'break']] + self.configurator.parser.add_server_directives(vhost, + rewrite_directive, replace=False, insert_at_top=True) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 5497f7e63..fbd6c0ade 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -276,7 +276,7 @@ class NginxParser(object): return False - def add_server_directives(self, vhost, directives, replace): + def add_server_directives(self, vhost, directives, replace, insert_at_top=False): """Add or replace directives in the server block identified by vhost. This method modifies vhost to be fully consistent with the new directives. @@ -293,10 +293,12 @@ class NginxParser(object): whose information we use to match on :param list directives: The directives to add :param bool replace: Whether to only replace existing directives + :param bool insert_at_top: True if the directives need to be inserted at the top + of the server block instead of the bottom """ self._modify_server_directives(vhost, - functools.partial(_add_directives, directives, replace)) + functools.partial(_add_directives, directives, replace, insert_at_top)) def remove_server_directives(self, vhost, directive_name, match_func=None): """Remove all directives of type directive_name. @@ -521,7 +523,7 @@ def _is_ssl_on_directive(entry): len(entry) == 2 and entry[0] == 'ssl' and entry[1] == 'on') -def _add_directives(directives, replace, block): +def _add_directives(directives, replace, insert_at_top, block): """Adds or replaces directives in a config block. When replace=False, it's an error to try and add a nonrepeatable directive that already @@ -535,17 +537,18 @@ def _add_directives(directives, replace, block): :param list directives: The new directives. :param bool replace: Described above. + :param bool insert_at_top: Described above. :param list block: The block to replace in """ for directive in directives: - _add_directive(block, directive, replace) + _add_directive(block, directive, replace, insert_at_top) if block and '\n' not in block[-1]: # could be " \n " or ["\n"] ! block.append(nginxparser.UnspacedList('\n')) INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location']) +REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location', 'rewrite']) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] @@ -597,7 +600,7 @@ def _find_location(block, directive_name, match_func=None): return next((index for index, line in enumerate(block) \ if line and line[0] == directive_name and (match_func is None or match_func(line))), None) -def _add_directive(block, directive, replace): +def _add_directive(block, directive, replace, insert_at_top): """Adds or replaces a single directive in a config block. See _add_directives for more documentation. @@ -619,7 +622,7 @@ def _add_directive(block, directive, replace): block[location] = directive comment_directive(block, location) return - # Append directive. Fail if the name is not a repeatable directive name, + # Append or prepend directive. Fail if the name is not a repeatable directive name, # and there is already a copy of that directive with a different value # in the config file. @@ -652,8 +655,15 @@ def _add_directive(block, directive, replace): _comment_out_directive(block, included_dir_loc, directive[1]) if can_append(location, directive_name): - block.append(directive) - comment_directive(block, len(block) - 1) + if insert_at_top: + # Add a newline so the comment doesn't comment + # out existing directives + block.insert(0, nginxparser.UnspacedList('\n')) + block.insert(0, directive) + comment_directive(block, 0) + else: + block.append(directive) + comment_directive(block, len(block) - 1) elif block[location] != directive: raise errors.MisconfigurationError(err_fmt.format(directive, block[location])) From e9b57e17833e19d36158ee7c1f9b7c684f118027 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 17 Jan 2018 08:02:10 -0800 Subject: [PATCH 17/98] Add (nonexistent) document root so we don't use the default value (#5437) --- certbot-nginx/certbot_nginx/http_01.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 2885e7ac0..c0dec061a 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -155,10 +155,15 @@ class NginxHttp01(common.ChallengePerformer): addrs = self._default_listen_addresses() block = [['listen', ' ', addr.to_string(include_default=False)] for addr in addrs] + # Ensure we 404 on any other request by setting a root + document_root = os.path.join( + self.configurator.config.work_dir, "http_01_nonexistent") + validation = achall.validation(achall.account_key) validation_path = self._get_validation_path(achall) block.extend([['server_name', ' ', achall.domain], + ['root', ' ', document_root], [['location', ' ', '=', ' ', validation_path], [['default_type', ' ', 'text/plain'], ['return', ' ', '200', ' ', validation]]]]) From bd231a3855c03d2943a2e3fe117605b4962726a8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 09:27:36 -0800 Subject: [PATCH 18/98] Error without vhosts and fix tests token type --- certbot-apache/certbot_apache/http_01.py | 20 ++++++++++++++++--- .../certbot_apache/tests/http_01_test.py | 9 ++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 2b2b8e796..4aa3215e6 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -2,6 +2,8 @@ import logging import os +from certbot import errors + from certbot.plugins import common logger = logging.getLogger(__name__) @@ -79,9 +81,8 @@ class ApacheHttp01(common.TLSSNI01): if vh: self._set_up_include_directive(vh) else: - for vh in self.configurator.vhosts: - if not vh.ssl: - self._set_up_include_directive(vh) + for vh in self._relevant_vhosts(): + self._set_up_include_directive(vh) self.configurator.reverter.register_file_creation( True, self.challenge_conf) @@ -97,6 +98,19 @@ class ApacheHttp01(common.TLSSNI01): with open(self.challenge_conf, "w") as new_conf: new_conf.write(config_text) + def _relevant_vhosts(self): + http01_port = str(self.configurator.config.http01_port) + relevant_vhosts = [] + for vhost in self.configurator.vhosts: + if any(a.is_wildcard() or a.get_port() == http01_port for a in vhost.addrs): + if not vhost.ssl: + relevant_vhosts.append(vhost) + if not relevant_vhosts: + raise errors.PluginError( + "Unable to find a virtual host listening on port {0}." + " Please add one.".format(http01_port)) + + return relevant_vhosts def _set_up_challenges(self): if not os.path.isdir(self.challenge_dir): diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 12f571354..652b65a09 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -6,6 +6,7 @@ import unittest from acme import challenges from certbot import achallenges +from certbot import errors from certbot.tests import acme_util @@ -63,7 +64,7 @@ class ApacheHttp01Test(util.ApacheTest): self.achalls.append( achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( - challenges.HTTP01(token=((chr(ord('a') + i) * 16))), + challenges.HTTP01(token=((chr(ord('a') + i).encode() * 16))), "pending"), domain=domain, account_key=self.account_key)) @@ -139,6 +140,12 @@ class ApacheHttp01Test(util.ApacheTest): domain="something.nonexistent", account_key=self.account_key)] self.common_perform_test(achalls, vhosts) + def test_no_vhost(self): + for achall in self.achalls: + self.http.add_chall(achall) + self.config.config.http01_port = 12345 + self.assertRaises(errors.PluginError, self.http.perform) + def common_perform_test(self, achalls, vhosts): """Tests perform with the given achalls.""" challenge_dir = self.http.challenge_dir From 63136be2e55473a12a0de36ec6eaa72e9cd0ab70 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 17 Jan 2018 20:07:38 +0200 Subject: [PATCH 19/98] Make sure the HTTP tests do not use wrong vhosts for asserts --- certbot-apache/certbot_apache/tests/http_01_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 12f571354..af32697d1 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -161,7 +161,8 @@ class ApacheHttp01Test(util.ApacheTest): matches = self.config.parser.find_dir("Include", self.http.challenge_conf, vhost.path) - self.assertEqual(len(matches), 1) + if not vhost.ssl: + self.assertEqual(len(matches), 1) self.assertTrue(os.path.exists(challenge_dir)) From 522532dc300d4f6921320d1a56167b5b6372b562 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 10:33:51 -0800 Subject: [PATCH 20/98] Improve no vhost error message --- certbot-apache/certbot_apache/http_01.py | 6 ++++-- certbot-apache/certbot_apache/tests/http_01_test.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 4aa3215e6..e463f3880 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -107,8 +107,10 @@ class ApacheHttp01(common.TLSSNI01): relevant_vhosts.append(vhost) if not relevant_vhosts: raise errors.PluginError( - "Unable to find a virtual host listening on port {0}." - " Please add one.".format(http01_port)) + "Unable to find a virtual host listening on port {0} which is" + " currently needed for Certbot to prove to the CA that you" + " control your domain. Please add a virtual host for port" + " {0}.".format(http01_port)) return relevant_vhosts diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 00e05e9fc..ee8566396 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -165,10 +165,10 @@ class ApacheHttp01Test(util.ApacheTest): self._test_challenge_file(achall) for vhost in vhosts: - matches = self.config.parser.find_dir("Include", - self.http.challenge_conf, - vhost.path) if not vhost.ssl: + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf, + vhost.path) self.assertEqual(len(matches), 1) self.assertTrue(os.path.exists(challenge_dir)) From 1bb2cfadf7d18ee01f70ebb3b34cedf356946837 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 15:34:34 -0800 Subject: [PATCH 21/98] hardcode vhosts and names for test (#5444) --- .../certbot_apache/tests/http_01_test.py | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index ee8566396..64a76649a 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -45,28 +45,19 @@ class ApacheHttp01Test(util.ApacheTest): self.account_key = self.rsa512jwk self.achalls = [] - self.vhosts = [] - vhost_index = 0 - for i in range(NUM_ACHALLS): - domain = None - # Find a vhost with a name/alias we can use - for j in range(vhost_index + 1, len(self.config.vhosts)): - vhost = self.config.vhosts[j] - domain = vhost.name if vhost.name else next(iter(vhost.aliases), None) - if domain: - self.vhosts.append(vhost) - vhost_index = j + 1 - break - else: # pragma: no cover - # If we didn't find a domain, we shouldn't continue the test. - self.fail("No usable vhost found") + vh_truth = util.get_vh_truth( + self.temp_dir, "debian_apache_2_4/multiple_vhosts") + # Takes the vhosts for encryption-example.demo, certbot.demo, and + # vhost.in.rootconf + self.vhosts = [vh_truth[0], vh_truth[3], vh_truth[10]] + for i in range(NUM_ACHALLS): self.achalls.append( achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( challenges.HTTP01(token=((chr(ord('a') + i).encode() * 16))), "pending"), - domain=domain, account_key=self.account_key)) + domain=self.vhosts[i].name, account_key=self.account_key)) modules = ["rewrite", "authz_core", "authz_host"] for mod in modules: From bf695d048dd1d3b14b1d521b5940d3c9a635fd64 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 15:55:29 -0800 Subject: [PATCH 22/98] Release 0.21.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 307 +++++++++++++----- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 10 +- letsencrypt-auto | 307 +++++++++++++----- letsencrypt-auto-source/certbot-auto.asc | 14 +- letsencrypt-auto-source/letsencrypt-auto | 26 +- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 +- 22 files changed, 494 insertions(+), 224 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 7ebdfe46e..a3e5fd744 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 3270f2c79..6a874c813 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-auto b/certbot-auto index 444bee1b9..558c330b2 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.20.0" +LE_AUTO_VERSION="0.21.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -68,10 +68,12 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive|renew) - ASSUME_YES=1;; + --noninteractive|--non-interactive) + NONINTERACTIVE=1;; --quiet) QUIET=1;; + renew) + ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -93,7 +95,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - ASSUME_YES=1 + NONINTERACTIVE=1 HELP=0 fi @@ -244,23 +246,42 @@ DeprecationBootstrap() { fi } - +MIN_PYTHON_VERSION="2.6" +MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version DeterminePythonVersion() { - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - if [ "$?" != "0" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 + # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + # + # If no Python is found, PYVER is set to 0. + if [ "$USE_PYTHON_3" = 1 ]; then + for LE_PYTHON in "$LE_PYTHON" python3; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + else + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + fi + if [ "$?" != "0" ]; then + if [ "$1" != "NOCRASH" ]; then + error "Cannot find any Pythons; please install one!" + exit 1 + else + PYVER=0 + return 0 + fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ "$PYVER" -lt 26 ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version 2.6." - exit 1 + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + if [ "$1" != "NOCRASH" ]; then + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." + exit 1 + fi fi } @@ -384,23 +405,19 @@ BootstrapDebCommon() { fi } -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 (EPEL must be installed manually) +# If new packages are installed by BootstrapRpmCommonBase below, version +# numbers in rpm_common.sh and rpm_python3.sh must be increased. +# Sets TOOL to the name of the package manager +# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. +# Enables EPEL if applicable and possible. +InitializeRPMCommonBase() { if type dnf 2>/dev/null then - tool=dnf + TOOL=dnf elif type yum 2>/dev/null then - tool=yum + TOOL=yum else error "Neither yum nor dnf found. Aborting bootstrap!" @@ -408,15 +425,15 @@ BootstrapRpmCommon() { fi if [ "$ASSUME_YES" = 1 ]; then - yes_flag="-y" + YES_FLAG="-y" fi if [ "$QUIET" = 1 ]; then QUIET_FLAG='--quiet' fi - if ! $tool list *virtualenv >/dev/null 2>&1; then + if ! $TOOL list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $tool list epel-release >/dev/null 2>&1; then + if ! $TOOL list epel-release >/dev/null 2>&1; then error "Enable the EPEL repository and try running Certbot again." exit 1 fi @@ -425,14 +442,20 @@ BootstrapRpmCommon() { sleep 1s /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..." sleep 1s fi - if ! $tool install $yes_flag $QUIET_FLAG epel-release; then + if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase # This call is superfluous in practice pkgs=" gcc @@ -444,10 +467,39 @@ BootstrapRpmCommon() { ca-certificates " - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $tool list python >/dev/null 2>&1; then + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then pkgs="$pkgs - python + mod_ssl + " + fi + + if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} + +# If new packages are installed by BootstrapRpmCommon below, this version +# number must be increased. +BOOTSTRAP_RPM_COMMON_VERSION=1 + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 + + InitializeRPMCommonBase + + # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. + if $TOOL list python >/dev/null 2>&1; then + python_pkgs="$python python-devel python-virtualenv python-tools @@ -455,9 +507,8 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $tool list python2 >/dev/null 2>&1; then - pkgs="$pkgs - python2 + elif $TOOL list python2 >/dev/null 2>&1; then + python_pkgs="$python2 python2-libs python2-setuptools python2-devel @@ -468,8 +519,7 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - pkgs="$pkgs - python27 + python_pkgs="$python27 python27-devel python27-virtualenv python27-tools @@ -477,16 +527,31 @@ BootstrapRpmCommon() { " fi - if $tool list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi + BootstrapRpmCommonBase "$python_pkgs" +} - if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" +# If new packages are installed by BootstrapRpmPython3 below, this version +# number must be increased. +BOOTSTRAP_RPM_PYTHON3_VERSION=1 + +BootstrapRpmPython3() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + # EPEL uses python34 + if $TOOL list python34 >/dev/null 2>&1; then + python_pkgs="python34 + python34-devel + python34-tools + " + else + error "No supported Python package available to install. Aborting bootstrap!" exit 1 fi + + BootstrapRpmCommonBase "$python_pkgs" } # If new packages are installed by BootstrapSuseCommon below, this version @@ -715,11 +780,27 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + # Run DeterminePythonVersion to decide on the basis of available Python versions + # whether to use 2.x or 3.x on RedHat-like systems. + # Then, revert LE_PYTHON to its previous state. + prev_le_python="$LE_PYTHON" + unset LE_PYTHON + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -816,7 +897,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -824,14 +909,26 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -841,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -858,10 +959,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -983,9 +1092,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1062,10 +1178,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1078,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.20.0 \ - --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ - --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 -acme==0.20.0 \ - --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ - --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 -certbot-apache==0.20.0 \ - --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ - --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f -certbot-nginx==0.20.0 \ - --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ - --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae +certbot==0.21.0 \ + --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ + --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 +acme==0.21.0 \ + --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ + --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d +certbot-apache==0.21.0 \ + --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ + --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b +certbot-nginx==0.21.0 \ + --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ + --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1319,9 +1431,10 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 @@ -1353,17 +1466,22 @@ On failure, return non-zero. """ -from __future__ import print_function +from __future__ import print_function, unicode_literals from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re +import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -from urllib2 import build_opener, HTTPHandler, HTTPSHandler -from urllib2 import HTTPError, URLError +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -1385,8 +1503,11 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. + if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): + self._opener = build_opener(HTTPSHandler(context=cert_none_context())) + else: + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -1408,7 +1529,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'w') as file: + with open(join(dir, filename), 'wb') as file: file.write(contents) @@ -1416,13 +1537,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json'))) + 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() + in metadata['releases'].keys() if re.match('^[0-9.]+$', r))) @@ -1439,7 +1560,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY, temp_dir, 'public_key.pem') + write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -1454,6 +1575,14 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) +def cert_none_context(): + """Create a SSLContext object to not check hostname.""" + # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + return context + + def main(): get = HttpsGetter().get flag = argv[1] @@ -1475,8 +1604,10 @@ if __name__ == '__main__': UNLIKELY_EOF # --------------------------------------------------------------------------- - DeterminePythonVersion - if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." + elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 1faf30643..1015fec47 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 428271045..f7f1b0b38 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 4a103193f..cd6f9458e 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 23098d4b6..35659acac 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 4ed5a06ca..447a1c1f4 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 8a0b88aab..0def6d7cb 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index b00bd1ac3..d51b9271b 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index b8f50254e..35c018706 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 2a388e487..027b3bbae 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 78007afb5..5fb54922a 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 7d1eb0bc9..882834852 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' install_requires = [ 'acme=={0}'.format(version), diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 2ad7aaf08..b5585bf42 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.21.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index cbea701ee..b0a212646 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.21.0.dev0' +__version__ = '0.21.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index abaa95b9b..f7318f0b3 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,7 +107,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.20.0 (certbot; + "". (default: CertbotACMEClient/0.21.0 (certbot; Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags encoded in the user agent are: --duplicate, --force- @@ -331,6 +331,14 @@ revoke: --reason {unspecified,keycompromise,affiliationchanged,superseded,cessationofoperation} Specify reason for revoking certificate. (default: unspecified) + --delete-after-revoke + Delete certificates after revoking them. (default: + None) + --no-delete-after-revoke + Do not delete certificates after revoking them. This + option should be used with caution because the 'renew' + subcommand will attempt to renew undeleted revoked + certificates. (default: None) register: Options for account registration & modification diff --git a/letsencrypt-auto b/letsencrypt-auto index 444bee1b9..558c330b2 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.20.0" +LE_AUTO_VERSION="0.21.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -68,10 +68,12 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive|renew) - ASSUME_YES=1;; + --noninteractive|--non-interactive) + NONINTERACTIVE=1;; --quiet) QUIET=1;; + renew) + ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -93,7 +95,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - ASSUME_YES=1 + NONINTERACTIVE=1 HELP=0 fi @@ -244,23 +246,42 @@ DeprecationBootstrap() { fi } - +MIN_PYTHON_VERSION="2.6" +MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version DeterminePythonVersion() { - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - if [ "$?" != "0" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 + # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + # + # If no Python is found, PYVER is set to 0. + if [ "$USE_PYTHON_3" = 1 ]; then + for LE_PYTHON in "$LE_PYTHON" python3; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + else + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + fi + if [ "$?" != "0" ]; then + if [ "$1" != "NOCRASH" ]; then + error "Cannot find any Pythons; please install one!" + exit 1 + else + PYVER=0 + return 0 + fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ "$PYVER" -lt 26 ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version 2.6." - exit 1 + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + if [ "$1" != "NOCRASH" ]; then + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." + exit 1 + fi fi } @@ -384,23 +405,19 @@ BootstrapDebCommon() { fi } -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 (EPEL must be installed manually) +# If new packages are installed by BootstrapRpmCommonBase below, version +# numbers in rpm_common.sh and rpm_python3.sh must be increased. +# Sets TOOL to the name of the package manager +# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. +# Enables EPEL if applicable and possible. +InitializeRPMCommonBase() { if type dnf 2>/dev/null then - tool=dnf + TOOL=dnf elif type yum 2>/dev/null then - tool=yum + TOOL=yum else error "Neither yum nor dnf found. Aborting bootstrap!" @@ -408,15 +425,15 @@ BootstrapRpmCommon() { fi if [ "$ASSUME_YES" = 1 ]; then - yes_flag="-y" + YES_FLAG="-y" fi if [ "$QUIET" = 1 ]; then QUIET_FLAG='--quiet' fi - if ! $tool list *virtualenv >/dev/null 2>&1; then + if ! $TOOL list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $tool list epel-release >/dev/null 2>&1; then + if ! $TOOL list epel-release >/dev/null 2>&1; then error "Enable the EPEL repository and try running Certbot again." exit 1 fi @@ -425,14 +442,20 @@ BootstrapRpmCommon() { sleep 1s /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..." sleep 1s fi - if ! $tool install $yes_flag $QUIET_FLAG epel-release; then + if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase # This call is superfluous in practice pkgs=" gcc @@ -444,10 +467,39 @@ BootstrapRpmCommon() { ca-certificates " - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $tool list python >/dev/null 2>&1; then + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then pkgs="$pkgs - python + mod_ssl + " + fi + + if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} + +# If new packages are installed by BootstrapRpmCommon below, this version +# number must be increased. +BOOTSTRAP_RPM_COMMON_VERSION=1 + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 + + InitializeRPMCommonBase + + # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. + if $TOOL list python >/dev/null 2>&1; then + python_pkgs="$python python-devel python-virtualenv python-tools @@ -455,9 +507,8 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $tool list python2 >/dev/null 2>&1; then - pkgs="$pkgs - python2 + elif $TOOL list python2 >/dev/null 2>&1; then + python_pkgs="$python2 python2-libs python2-setuptools python2-devel @@ -468,8 +519,7 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - pkgs="$pkgs - python27 + python_pkgs="$python27 python27-devel python27-virtualenv python27-tools @@ -477,16 +527,31 @@ BootstrapRpmCommon() { " fi - if $tool list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi + BootstrapRpmCommonBase "$python_pkgs" +} - if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" +# If new packages are installed by BootstrapRpmPython3 below, this version +# number must be increased. +BOOTSTRAP_RPM_PYTHON3_VERSION=1 + +BootstrapRpmPython3() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + # EPEL uses python34 + if $TOOL list python34 >/dev/null 2>&1; then + python_pkgs="python34 + python34-devel + python34-tools + " + else + error "No supported Python package available to install. Aborting bootstrap!" exit 1 fi + + BootstrapRpmCommonBase "$python_pkgs" } # If new packages are installed by BootstrapSuseCommon below, this version @@ -715,11 +780,27 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + # Run DeterminePythonVersion to decide on the basis of available Python versions + # whether to use 2.x or 3.x on RedHat-like systems. + # Then, revert LE_PYTHON to its previous state. + prev_le_python="$LE_PYTHON" + unset LE_PYTHON + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -816,7 +897,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -824,14 +909,26 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -841,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -858,10 +959,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -983,9 +1092,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1062,10 +1178,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1078,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.20.0 \ - --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ - --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 -acme==0.20.0 \ - --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ - --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 -certbot-apache==0.20.0 \ - --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ - --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f -certbot-nginx==0.20.0 \ - --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ - --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae +certbot==0.21.0 \ + --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ + --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 +acme==0.21.0 \ + --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ + --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d +certbot-apache==0.21.0 \ + --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ + --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b +certbot-nginx==0.21.0 \ + --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ + --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1319,9 +1431,10 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 @@ -1353,17 +1466,22 @@ On failure, return non-zero. """ -from __future__ import print_function +from __future__ import print_function, unicode_literals from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re +import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -from urllib2 import build_opener, HTTPHandler, HTTPSHandler -from urllib2 import HTTPError, URLError +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -1385,8 +1503,11 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. + if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): + self._opener = build_opener(HTTPSHandler(context=cert_none_context())) + else: + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -1408,7 +1529,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'w') as file: + with open(join(dir, filename), 'wb') as file: file.write(contents) @@ -1416,13 +1537,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json'))) + 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() + in metadata['releases'].keys() if re.match('^[0-9.]+$', r))) @@ -1439,7 +1560,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY, temp_dir, 'public_key.pem') + write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -1454,6 +1575,14 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) +def cert_none_context(): + """Create a SSLContext object to not check hostname.""" + # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + return context + + def main(): get = HttpsGetter().get flag = argv[1] @@ -1475,8 +1604,10 @@ if __name__ == '__main__': UNLIKELY_EOF # --------------------------------------------------------------------------- - DeterminePythonVersion - if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." + elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index eeab78cd6..9df8013a0 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- Version: GnuPG v2 -iQEcBAABCAAGBQJaKHMlAAoJEE0XyZXNl3Xy6OEH/iPg6D6+zco4NHMwxYIcTWVt -XE4u3CjuLcEVsvEnJYNSA48NHyi9rIqMHd+IneLU+lCG2D7eBsisNNyVPIgHktTf -p9i0WoZB+axe1glv9FJSZvjvr2d/ic4/wYHBF1c+szb9p8Z7o5Lhqa9/gtLJ/SZX -OGU0wok4hPIB6emq5zvmi/+r1AiOECXE26lZ0STp6wDkvz+ahTJSk6UaPCDY+Az4 -X2VmnRSks/gk7Q8cloFnyiPXyFMQHdGIBRrIXsSix90QqmNUF7iYb8sbHksU23EI -/LmIwSJlDm6KNOO2nllBB/uIg2ki7g0z7R4uf7XF4im+P95PAL/tQQ45lVj8DXE= -=Is56 +iQEcBAABCAAGBQJaX+JUAAoJEE0XyZXNl3XyUCkH/jowI7yayXREoBUWpLuByd/n +e1wGLQjnZYkxv/AJGJ63G3QvwpzmIqo3r/6K4ARlUcdOnepZRDpF6jC4F5q9vBwW +AvUVU2B7e6mC6l/jXNepS8xowEwkQptQBDfnqh8TTeTb3rQTFod8X41skZ2633HL +RX4ditKaGMbcswMn6+5/juz0YK5ujVdVTcMeMcZKP2tvPJ9Y08YdpY6IdrM0Mfhn +IqssjM06CzsiYHeNOXfRY4vAPw4Oq/md3bf6ZpPCee1HPiDm0NvHtTemWBkPIehf +yy0U8JIDIZha4WKo3yifbZFL5Zf5czVkrtqQ3DBRcLrCFtBh2aTVsIMJkpW/wFo= +=d/hS -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index b86517bb6..558c330b2 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0.dev0" +LE_AUTO_VERSION="0.21.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1190,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.20.0 \ - --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ - --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 -acme==0.20.0 \ - --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ - --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 -certbot-apache==0.20.0 \ - --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ - --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f -certbot-nginx==0.20.0 \ - --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ - --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae +certbot==0.21.0 \ + --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ + --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 +acme==0.21.0 \ + --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ + --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d +certbot-apache==0.21.0 \ + --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ + --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b +certbot-nginx==0.21.0 \ + --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ + --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index e276aae5353e8c5bdf78fc91950e60649baa241f..2e52f03fdf9a1fbe2049d89f071a583bcf919976 100644 GIT binary patch literal 256 zcmV+b0ssD{xCAf=x;KV2@d03ctAnF7mn+aqUc{p=)VXkyhWU!~cN;%G^8WY$S?ta1 z2hmQE4T`~+Zwq%*K~cAA%;XQy8S-ox4a$2d0@tMjK|wQYPjC$Cv}rUZ3(4Cmz7$?G zU;e#Fi7%+uTC73=Hbzf*EfB0nOu(b3_b6CNN5}h_$-t&E@K&w437B8*P8QflhULg&MO?mxD$RuiY1;Ik~L_O?oC&* zWM|7qv=mZh!YK!wmdDwpz@wj96nhJniJ_laAj^74?$6FZ`p5@HrirST)N_#L(FvXz GZCemMz+EWKAt%U!XWg-L;RUoOlOZsAFb4Ic}LGE#!b+HMXRo%-QglHm~xc z7rA=rPZ17(d9O44~^dV!y) z3$?Ok9e8=A1v3-fBldYu$u&26@w%#BftaFQ;t)7s73=9Vm2g;i>-a0vSdi>_B;&sD z-BgJaKSacB+~@MWWUpduDsx8T Date: Wed, 17 Jan 2018 15:55:41 -0800 Subject: [PATCH 23/98] Bump version to 0.22.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index a3e5fd744..ce426cf74 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 6a874c813..38f41e9f1 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 1015fec47..8f9f897cf 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index f7f1b0b38..612e7259f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index cd6f9458e..3157400c6 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 35659acac..1a68400fa 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 447a1c1f4..35de47308 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 0def6d7cb..a946d00a4 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index d51b9271b..8585fc848 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 35c018706..4fec37e29 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 027b3bbae..dca9ebf27 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 5fb54922a..bfa72b50b 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 882834852..8df687972 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' install_requires = [ 'acme=={0}'.format(version), diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b5585bf42..152f77de8 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index b0a212646..2869d29b0 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.21.0' +__version__ = '0.22.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 558c330b2..9c8ccf344 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0" +LE_AUTO_VERSION="0.22.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 103039ca40c3a300816c5a957a0bf13a033234de Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 17:46:56 -0800 Subject: [PATCH 24/98] Add 0.21.0 changelog --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4acfc0401..f6f7be9fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,51 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.21.0 - 2018-01-17 + +### Added + +* Support for the HTTP-01 challenge type was added to our Apache and Nginx + plugins. For those not aware, Let's Encrypt disabled the TLS-SNI-01 challenge + type which was what was previously being used by our Apache and Nginx plugins + last week due to a security issue. For more information about Let's Encrypt's + change, click + [here](https://community.letsencrypt.org/t/2018-01-11-update-regarding-acme-tls-sni-and-shared-hosting-infrastructure/50188). + Our Apache and Nginx plugins will automatically switch to use HTTP-01 so no + changes need to be made to your Certbot configuration, however, you should + make sure your server is accessible on port 80 and isn't behind an external + proxy doing things like redirecting all traffic from HTTP to HTTPS. HTTP to + HTTPS redirects inside Apache and Nginx are fine. +* IPv6 support was added to the Nginx plugin. +* Support for automatically creating server blocks based on the default server + block was added to the Nginx plugin. +* The flags --delete-after-revoke and --no-delete-after-revoke were added + allowing users to control whether the revoke subcommand also deletes the + certificates it is revoking. + +### Changed + +* We deprecated support for Python 2.6 and Python 3.3 in Certbot and its ACME + library. Support for these versions of Python will be removed in the next + major release of Certbot. If you are using certbot-auto on a RHEL 6 based + system, it will guide you through the process of installing Python 3. +* We split our implementation of JOSE (Javascript Object Signing and + Encryption) out of our ACME library and into a separate package named josepy. + This package is available on [PyPI](https://pypi.python.org/pypi/josepy) and + on [GitHub](https://github.com/certbot/josepy). +* We updated the ciphersuites used in Apache to the new [values recommended by + Mozilla](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29). + The major change here is adding ChaCha20 to the list of supported + ciphersuites. + +### Fixed + +* An issue with our Apache plugin on Gentoo due to differences in their + apache2ctl command have been resolved. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/47?closed=1 + ## 0.20.0 - 2017-12-06 ### Added From b0aa8b7c0bcddb0837d9a57d021152bc05473b8e Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 24 Jan 2018 02:46:36 +0200 Subject: [PATCH 25/98] Work around Basic Authentication for challenge dir in Apache (#5461) Unfortunately, the way that Apache merges the configuration directives is different for mod_rewrite and / directives. To work around basic auth in VirtualHosts, the challenge override Include had to be split in two. The first part handles overrides for RewriteRule and the other part will handle overrides for and directives. --- certbot-apache/certbot_apache/http_01.py | 60 +++++++++++++------ .../certbot_apache/tests/http_01_test.py | 24 +++++--- 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index e463f3880..cce93a646 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -11,30 +11,43 @@ logger = logging.getLogger(__name__) class ApacheHttp01(common.TLSSNI01): """Class that performs HTTP-01 challenges within the Apache configurator.""" - CONFIG_TEMPLATE22 = """\ + CONFIG_TEMPLATE22_PRE = """\ RewriteEngine on RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L] + """ + CONFIG_TEMPLATE22_POST = """\ Order Allow,Deny Allow from all + + Order Allow,Deny + Allow from all + """ - CONFIG_TEMPLATE24 = """\ + CONFIG_TEMPLATE24_PRE = """\ RewriteEngine on RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END] - + """ + CONFIG_TEMPLATE24_POST = """\ Require all granted + + Require all granted + """ def __init__(self, *args, **kwargs): super(ApacheHttp01, self).__init__(*args, **kwargs) - self.challenge_conf = os.path.join( + self.challenge_conf_pre = os.path.join( self.configurator.conf("challenge-location"), - "le_http_01_challenge.conf") + "le_http_01_challenge_pre.conf") + self.challenge_conf_post = os.path.join( + self.configurator.conf("challenge-location"), + "le_http_01_challenge_post.conf") self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") @@ -79,24 +92,32 @@ class ApacheHttp01(common.TLSSNI01): chall.domain, filter_defaults=False, port=str(self.configurator.config.http01_port)) if vh: - self._set_up_include_directive(vh) + self._set_up_include_directives(vh) else: for vh in self._relevant_vhosts(): - self._set_up_include_directive(vh) + self._set_up_include_directives(vh) self.configurator.reverter.register_file_creation( - True, self.challenge_conf) + True, self.challenge_conf_pre) + self.configurator.reverter.register_file_creation( + True, self.challenge_conf_post) if self.configurator.version < (2, 4): - config_template = self.CONFIG_TEMPLATE22 + config_template_pre = self.CONFIG_TEMPLATE22_PRE + config_template_post = self.CONFIG_TEMPLATE22_POST else: - config_template = self.CONFIG_TEMPLATE24 + config_template_pre = self.CONFIG_TEMPLATE24_PRE + config_template_post = self.CONFIG_TEMPLATE24_POST - config_text = config_template.format(self.challenge_dir) + config_text_pre = config_template_pre.format(self.challenge_dir) + config_text_post = config_template_post.format(self.challenge_dir) - logger.debug("writing a config file with text:\n %s", config_text) - with open(self.challenge_conf, "w") as new_conf: - new_conf.write(config_text) + logger.debug("writing a pre config file with text:\n %s", config_text_pre) + with open(self.challenge_conf_pre, "w") as new_conf: + new_conf.write(config_text_pre) + logger.debug("writing a post config file with text:\n %s", config_text_post) + with open(self.challenge_conf_post, "w") as new_conf: + new_conf.write(config_text_post) def _relevant_vhosts(self): http01_port = str(self.configurator.config.http01_port) @@ -137,14 +158,17 @@ class ApacheHttp01(common.TLSSNI01): return response - def _set_up_include_directive(self, vhost): - """Includes override configuration to the beginning of VirtualHost. - Note that this include isn't added to Augeas search tree""" + def _set_up_include_directives(self, vhost): + """Includes override configuration to the beginning and to the end of + VirtualHost. Note that this include isn't added to Augeas search tree""" if vhost not in self.moded_vhosts: logger.debug( "Adding a temporary challenge validation Include for name: %s " + "in: %s", vhost.name, vhost.filep) self.configurator.parser.add_dir_beginning( - vhost.path, "Include", self.challenge_conf) + vhost.path, "Include", self.challenge_conf_pre) + self.configurator.parser.add_dir( + vhost.path, "Include", self.challenge_conf_post) + self.moded_vhosts.add(vhost) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 64a76649a..9ed4ee509 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -158,23 +158,31 @@ class ApacheHttp01Test(util.ApacheTest): for vhost in vhosts: if not vhost.ssl: matches = self.config.parser.find_dir("Include", - self.http.challenge_conf, + self.http.challenge_conf_pre, + vhost.path) + self.assertEqual(len(matches), 1) + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf_post, vhost.path) self.assertEqual(len(matches), 1) self.assertTrue(os.path.exists(challenge_dir)) def _test_challenge_conf(self): - with open(self.http.challenge_conf) as f: - conf_contents = f.read() + with open(self.http.challenge_conf_pre) as f: + pre_conf_contents = f.read() - self.assertTrue("RewriteEngine on" in conf_contents) - self.assertTrue("RewriteRule" in conf_contents) - self.assertTrue(self.http.challenge_dir in conf_contents) + with open(self.http.challenge_conf_post) as f: + post_conf_contents = f.read() + + self.assertTrue("RewriteEngine on" in pre_conf_contents) + self.assertTrue("RewriteRule" in pre_conf_contents) + + self.assertTrue(self.http.challenge_dir in post_conf_contents) if self.config.version < (2, 4): - self.assertTrue("Allow from all" in conf_contents) + self.assertTrue("Allow from all" in post_conf_contents) else: - self.assertTrue("Require all granted" in conf_contents) + self.assertTrue("Require all granted" in post_conf_contents) def _test_challenge_file(self, achall): name = os.path.join(self.http.challenge_dir, achall.chall.encode("token")) From c0068791ce2a4edbe4b7293c6eab21d8b62347ca Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 24 Jan 2018 13:56:40 -0800 Subject: [PATCH 26/98] add let's encrypt status to footer and fix link --- docs/_templates/footer.html | 52 +++++++++++++++++++++++++++++++++++++ docs/conf.py | 3 ++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 docs/_templates/footer.html diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html new file mode 100644 index 000000000..8fd0f127d --- /dev/null +++ b/docs/_templates/footer.html @@ -0,0 +1,52 @@ +
+ {% if (theme_prev_next_buttons_location == 'bottom' or theme_prev_next_buttons_location == 'both') and (next or prev) %} + + {% endif %} + +
+ +
+

+ + © Copyright 2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license. + +
+
+ + Let's Encrypt Status + + + {%- if build_id and build_url %} + {% trans build_url=build_url, build_id=build_id %} + + Build + {{ build_id }}. + + {% endtrans %} + {%- elif commit %} + {% trans commit=commit %} + + Revision {{ commit }}. + + {% endtrans %} + {%- elif last_updated %} + {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} + {%- endif %} + +

+
+ + {%- if show_sphinx %} + {% trans %}Built with Sphinx using a theme provided by Read the Docs{% endtrans %}. + {%- endif %} + + {%- block extrafooter %} {% endblock %} + +
diff --git a/docs/conf.py b/docs/conf.py index 73df47dbd..09bb44285 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,7 +64,8 @@ master_doc = 'index' # General information about the project. project = u'Certbot' -copyright = u'2014-2016 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license ' +# this is now overridden by the footer.html template +#copyright = u'2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 0a4f926b160cf118c81974e91df7f196bc7a379d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 24 Jan 2018 15:01:42 -0800 Subject: [PATCH 27/98] Remove Default Detector log line. (#5372) This produces a super-long log line that wraps to 30-60 lines, depending on screen width. Even though it's just at debug level, it clutters up the integration test output without providing proportional debugging value. * Remove Default Detector log line. This produces about 30 lines of log output. Even though it's just at debug level, it clutters up the integration test output without providing proportional debugging value. * Add more useful logs. --- certbot/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index f0fa7eb7e..dec7474f9 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -207,13 +207,15 @@ def set_by_cli(var): # propagate plugin requests: eg --standalone modifies config.authenticator detector.authenticator, detector.installer = ( plugin_selection.cli_plugin_requests(detector)) - logger.debug("Default Detector is %r", detector) if not isinstance(getattr(detector, var), _Default): + logger.debug("Var %s=%s (set by user).", var, getattr(detector, var)) return True for modifier in VAR_MODIFIERS.get(var, []): if set_by_cli(modifier): + logger.debug("Var %s=%s (set by user).", + var, VAR_MODIFIERS.get(var, [])) return True return False From 8a9f21cdd36fb9a424430f9c807bacb27c1cfbf4 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 24 Jan 2018 22:19:32 -0800 Subject: [PATCH 28/98] Fix Nginx redirect issue (#5479) * wrap redirect in if host matches * return 404 if we've created a new block * change domain matching to exact match * insert new redirect directive at the top * add a redirect block to the top if it doesn't already exist, even if there's an existing redirect * fix obj tests * remove active parameter * update tests * add back spaces * move imports * remove unused code --- certbot-nginx/certbot_nginx/configurator.py | 50 ++++---- certbot-nginx/certbot_nginx/obj.py | 21 ---- .../certbot_nginx/tests/configurator_test.py | 110 +++--------------- certbot-nginx/certbot_nginx/tests/obj_test.py | 14 +-- 4 files changed, 44 insertions(+), 151 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index bb2933a39..9f091c0fd 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -31,16 +31,6 @@ from certbot_nginx import http_01 logger = logging.getLogger(__name__) -REDIRECT_BLOCK = [ - ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], - ['\n'] -] - -REDIRECT_COMMENT_BLOCK = [ - ['\n ', '#', ' Redirect non-https traffic to https'], - ['\n ', '#', ' return 301 https://$host$request_uri;'], - ['\n'] -] @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) @@ -571,24 +561,17 @@ class NginxConfigurator(common.Installer): logger.warning("Failed %s for %s", enhancement, domain) raise - def _has_certbot_redirect(self, vhost): - test_redirect_block = _test_block_from_block(REDIRECT_BLOCK) + def _has_certbot_redirect(self, vhost, domain): + test_redirect_block = _test_block_from_block(_redirect_block_for_domain(domain)) return vhost.contains_list(test_redirect_block) - def _has_certbot_redirect_comment(self, vhost): - test_redirect_comment_block = _test_block_from_block(REDIRECT_COMMENT_BLOCK) - return vhost.contains_list(test_redirect_comment_block) - - def _add_redirect_block(self, vhost, active=True): + def _add_redirect_block(self, vhost, domain): """Add redirect directive to vhost """ - if active: - redirect_block = REDIRECT_BLOCK - else: - redirect_block = REDIRECT_COMMENT_BLOCK + redirect_block = _redirect_block_for_domain(domain) self.parser.add_server_directives( - vhost, redirect_block, replace=False) + vhost, redirect_block, replace=False, insert_at_top=True) def _enable_redirect(self, domain, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. @@ -615,6 +598,7 @@ class NginxConfigurator(common.Installer): self.DEFAULT_LISTEN_PORT) return + new_vhost = None if vhost.ssl: new_vhost = self.parser.duplicate_vhost(vhost, only_directives=['listen', 'server_name']) @@ -631,20 +615,18 @@ class NginxConfigurator(common.Installer): # remove all non-ssl addresses from the existing block self.parser.remove_server_directives(vhost, 'listen', match_func=_no_ssl_match_func) + # Add this at the bottom to get the right order of directives + return_404_directive = [['\n ', 'return', ' ', '404']] + self.parser.add_server_directives(new_vhost, return_404_directive, replace=False) + vhost = new_vhost - if self._has_certbot_redirect(vhost): + if self._has_certbot_redirect(vhost, domain): logger.info("Traffic on port %s already redirecting to ssl in %s", self.DEFAULT_LISTEN_PORT, vhost.filep) - elif vhost.has_redirect(): - if not self._has_certbot_redirect_comment(vhost): - self._add_redirect_block(vhost, active=False) - logger.info("The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.", vhost.filep) else: # Redirect plaintextish host to https - self._add_redirect_block(vhost, active=True) + self._add_redirect_block(vhost, domain) logger.info("Redirecting all traffic on port %s to ssl in %s", self.DEFAULT_LISTEN_PORT, vhost.filep) @@ -907,6 +889,14 @@ def _test_block_from_block(block): parser.comment_directive(test_block, 0) return test_block[:-1] +def _redirect_block_for_domain(domain): + redirect_block = [[ + ['\n ', 'if', ' ', '($host', ' ', '=', ' ', '%s)' % domain, ' '], + [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + '\n ']], + ['\n']] + return redirect_block + def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index f5ac5c2e3..e8dc8936d 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -193,15 +193,6 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return False - def has_redirect(self): - """Determine if this vhost has a redirecting statement - """ - for directive_name in REDIRECT_DIRECTIVES: - found = _find_directive(self.raw, directive_name) - if found is not None: - return True - return False - def contains_list(self, test): """Determine if raw server block contains test list at top level """ @@ -225,15 +216,3 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods for a in self.addrs: if not a.ipv6: return True - -def _find_directive(directives, directive_name): - """Find a directive of type directive_name in directives - """ - if not directives or isinstance(directives, six.string_types) or len(directives) == 0: - return None - - if directives[0] == directive_name: - return directives - - matches = (_find_directive(line, directive_name) for line in directives) - return next((m for m in matches if m is not None), None) diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 7475df40c..acb7ee282 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -18,6 +18,8 @@ from certbot.tests import util as certbot_test_util from certbot_nginx import constants from certbot_nginx import obj from certbot_nginx import parser +from certbot_nginx.configurator import _redirect_block_for_domain +from certbot_nginx.nginxparser import UnspacedList from certbot_nginx.tests import util @@ -447,7 +449,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_redirect_enhance(self): # Test that we successfully add a redirect when there is # a listen directive - expected = ['return', '301', 'https://$host$request_uri'] + expected = UnspacedList(_redirect_block_for_domain("www.example.com"))[0] example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("www.example.com", "redirect") @@ -460,6 +462,8 @@ class NginxConfiguratorTest(util.NginxTest): migration_conf = self.config.parser.abs_path('sites-enabled/migration.com') self.config.enhance("migration.com", "redirect") + expected = UnspacedList(_redirect_block_for_domain("migration.com"))[0] + generated_conf = self.config.parser.parsed[migration_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) @@ -484,101 +488,27 @@ class NginxConfiguratorTest(util.NginxTest): ['ssl_dhparam', self.config.ssl_dhparams], ['#', ' managed by Certbot'], [], []]], [['server'], [ + [['if', '($host', '=', 'www.example.com)'], [ + ['return', '301', 'https://$host$request_uri']]], + ['#', ' managed by Certbot'], [], ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], ['server_name', 'example.*'], - ['return', '301', 'https://$host$request_uri'], ['#', ' managed by Certbot'], - [], []]]], + ['return', '404'], ['#', ' managed by Certbot'], [], [], []]]], generated_conf) @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): + def test_certbot_redirect_exists(self, mock_contains_list): # Test that we add no redirect statement if there is already a # redirect in the block that is managed by certbot # Has a certbot redirect - mock_has_redirect.return_value = True mock_contains_list.return_value = True with mock.patch("certbot_nginx.configurator.logger") as mock_logger: self.config.enhance("www.example.com", "redirect") self.assertEqual(mock_logger.info.call_args[0][0], "Traffic on port %s already redirecting to ssl in %s") - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_non_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): - # Test that we add a redirect as a comment if there is already a - # redirect-class statement in the block that isn't managed by certbot - example_conf = self.config.parser.abs_path('sites-enabled/example.com') - - # Has a non-Certbot redirect, and has no existing comment - mock_contains_list.return_value = False - mock_has_redirect.return_value = True - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertEqual(mock_logger.info.call_args[0][0], - "The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.") - generated_conf = self.config.parser.parsed[example_conf] - expected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' return 301 https://$host$request_uri;'], - ] - for line in expected: - self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) - - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_non_certbot_redirect_exists_has_ssl_copy(self, mock_has_redirect, mock_contains_list): - # Test that we add a redirect as a comment if there is already a - # redirect-class statement in the block that isn't managed by certbot - example_conf = self.config.parser.abs_path('sites-enabled/example.com') - - self.config.deploy_cert( - "example.org", - "example/cert.pem", - "example/key.pem", - "example/chain.pem", - "example/fullchain.pem") - - # Has a non-Certbot redirect, and has no existing comment - mock_contains_list.return_value = False - mock_has_redirect.return_value = True - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertEqual(mock_logger.info.call_args[0][0], - "The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.") - generated_conf = self.config.parser.parsed[example_conf] - expected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' return 301 https://$host$request_uri;'], - ] - for line in expected: - self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) - - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - @mock.patch('certbot_nginx.configurator.NginxConfigurator._has_certbot_redirect_comment') - @mock.patch('certbot_nginx.configurator.NginxConfigurator._add_redirect_block') - def test_redirect_comment_exists(self, mock_add_redirect_block, - mock_has_cb_redirect_comment, mock_has_redirect, mock_contains_list): - # Test that we add nothing if there is a non-Certbot redirect and a - # preexisting comment - # Has a non-Certbot redirect and a comment - mock_has_redirect.return_value = True - mock_contains_list.return_value = False # self._has_certbot_redirect(vhost): - mock_has_cb_redirect_comment.return_value = True - - # assert _add_redirect_block not called - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertFalse(mock_add_redirect_block.called) - self.assertTrue(mock_logger.info.called) - def test_redirect_dont_enhance(self): # Test that we don't accidentally add redirect to ssl-only block with mock.patch("certbot_nginx.configurator.logger") as mock_logger: @@ -586,22 +516,18 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(mock_logger.info.call_args[0][0], 'No matching insecure server blocks listening on port %s found.') - def test_no_double_redirect(self): - # Test that we don't also add the commented redirect if we've just added - # a redirect to that vhost this run + def test_double_redirect(self): + # Test that we add one redirect for each domain example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("example.com", "redirect") self.config.enhance("example.org", "redirect") - unexpected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' if ($scheme != "https") {'], - ['#', ' return 301 https://$host$request_uri;'], - ['#', ' } # managed by Certbot'] - ] + expected1 = UnspacedList(_redirect_block_for_domain("example.com"))[0] + expected2 = UnspacedList(_redirect_block_for_domain("example.org"))[0] + generated_conf = self.config.parser.parsed[example_conf] - for line in unexpected: - self.assertFalse(util.contains_at_depth(generated_conf, line, 2)) + self.assertTrue(util.contains_at_depth(generated_conf, expected1, 2)) + self.assertTrue(util.contains_at_depth(generated_conf, expected2, 2)) def test_staple_ocsp_bad_version(self): self.config.version = (1, 3, 1) @@ -763,7 +689,7 @@ class NginxConfiguratorTest(util.NginxTest): self.config.parser.load() - expected = ['return', '301', 'https://$host$request_uri'] + expected = UnspacedList(_redirect_block_for_domain("www.nomatch.com"))[0] generated_conf = self.config.parser.parsed[default_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index 92cb0e086..b30338b5b 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -162,17 +162,15 @@ class VirtualHostTest(unittest.TestCase): 'enabled: False']) self.assertEqual(stringified, str(self.vhost1)) - def test_has_redirect(self): - self.assertTrue(self.vhost1.has_redirect()) - self.assertTrue(self.vhost2.has_redirect()) - self.assertTrue(self.vhost3.has_redirect()) - self.assertFalse(self.vhost4.has_redirect()) - def test_contains_list(self): from certbot_nginx.obj import VirtualHost from certbot_nginx.obj import Addr - from certbot_nginx.configurator import REDIRECT_BLOCK, _test_block_from_block - test_needle = _test_block_from_block(REDIRECT_BLOCK) + from certbot_nginx.configurator import _test_block_from_block + test_block = [ + ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + ['\n'] + ] + test_needle = _test_block_from_block(test_block) test_haystack = [['listen', '80'], ['root', '/var/www/html'], ['index', 'index.html index.htm index.nginx-debian.html'], ['server_name', 'two.functorkitten.xyz'], ['listen', '443 ssl'], From a1aba5842e979379d98f3128d140d1270fb951c5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 24 Jan 2018 22:23:20 -0800 Subject: [PATCH 29/98] Fix --no-bootstrap on CentOS/RHEL 6 (#5476) * fix --no-bootstrap on RHEL6 * Add regression test --- letsencrypt-auto-source/letsencrypt-auto | 20 ++++++++++++------- .../letsencrypt-auto.template | 20 ++++++++++++------- .../tests/centos6_tests.sh | 8 ++++++++ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9c8ccf344..b5cac23e6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 96e5c2db0..2ce337002 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -300,13 +300,8 @@ DeterminePythonVersion() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -402,6 +397,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh index a0e96edf8..2c6dcf734 100644 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -69,5 +69,13 @@ fi echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present." echo "" +export VENV_PATH=$(mktemp -d) +"$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1 +if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1)" != 3 ]; then + echo "Python 3 wasn't used with --no-bootstrap!" + exit 1 +fi +unset VENV_PATH + # test using python3 pytest -v -s certbot/letsencrypt-auto-source/tests From a2239baa45de446ab246047171ad86e014bf580c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 24 Jan 2018 22:38:36 -0800 Subject: [PATCH 30/98] fix test_tests.sh (#5478) --- tests/letstest/scripts/test_tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index 4bed2dd3a..e6ab836b8 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -1,11 +1,13 @@ #!/bin/sh -xe +LE_AUTO="letsencrypt/letsencrypt-auto-source/letsencrypt-auto" +LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" MODULES="acme certbot certbot_apache certbot_nginx" VENV_NAME=venv # *-auto respects VENV_PATH -letsencrypt/certbot-auto --debug --os-packages-only --non-interactive -LE_AUTO_SUDO="" VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --no-bootstrap --non-interactive --version +$LE_AUTO --os-packages-only +LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version . $VENV_NAME/bin/activate # change to an empty directory to ensure CWD doesn't affect tests From 43bbaadd1154c0cd44eeb761bf33563953e793ae Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Jan 2018 15:29:38 -0800 Subject: [PATCH 31/98] Update certbot-auto and help (#5487) * Release 0.21.1 (cherry picked from commit ff60d70e68f7b4ddc60a61848190fbb6e55b5d2b) * Bump version to 0.22.0 --- certbot-auto | 46 ++++++++++-------- docs/cli-help.txt | 18 +++---- letsencrypt-auto | 46 ++++++++++-------- letsencrypt-auto-source/certbot-auto.asc | 16 +++--- letsencrypt-auto-source/letsencrypt-auto | 24 ++++----- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++----- 7 files changed, 92 insertions(+), 82 deletions(-) diff --git a/certbot-auto b/certbot-auto index 558c330b2..d3a5c23e5 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0" +LE_AUTO_VERSION="0.21.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -1190,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/docs/cli-help.txt b/docs/cli-help.txt index f7318f0b3..abebdb9c9 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,9 +107,9 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.21.0 (certbot; - Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags + "". (default: CertbotACMEClient/0.21.1 (certbot; + darwin 10.13.3) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags encoded in the user agent are: --duplicate, --force- renew, --allow-subset-of-names, -n, and whether any hooks are set. @@ -448,11 +448,9 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary. (default: - a2enmod) + Path to the Apache 'a2enmod' binary. (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary. (default: - a2dismod) + Path to the Apache 'a2dismod' binary. (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension. (default: -le- ssl.conf) @@ -466,13 +464,13 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration. (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for - you.(Only Ubuntu/Debian currently) (default: True) + you.(Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you.(Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 diff --git a/letsencrypt-auto b/letsencrypt-auto index 558c330b2..d3a5c23e5 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0" +LE_AUTO_VERSION="0.21.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -1190,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 9df8013a0..f28fd9893 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -Version: GnuPG v2 -iQEcBAABCAAGBQJaX+JUAAoJEE0XyZXNl3XyUCkH/jowI7yayXREoBUWpLuByd/n -e1wGLQjnZYkxv/AJGJ63G3QvwpzmIqo3r/6K4ARlUcdOnepZRDpF6jC4F5q9vBwW -AvUVU2B7e6mC6l/jXNepS8xowEwkQptQBDfnqh8TTeTb3rQTFod8X41skZ2633HL -RX4ditKaGMbcswMn6+5/juz0YK5ujVdVTcMeMcZKP2tvPJ9Y08YdpY6IdrM0Mfhn -IqssjM06CzsiYHeNOXfRY4vAPw4Oq/md3bf6ZpPCee1HPiDm0NvHtTemWBkPIehf -yy0U8JIDIZha4WKo3yifbZFL5Zf5czVkrtqQ3DBRcLrCFtBh2aTVsIMJkpW/wFo= -=d/hS +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlpqMlYACgkQTRfJlc2X +dfKHfQgAnZQJ34jFoVqEodT0EjvkFKZif4V/zXTsVwTHn107BcLCpH/9gjANrSo3 +JpvseH2q0odhOAZA4rZKH4Geh+5fsUl3Ew9YB28RXeyqEfCATUqPq6q+jAi55SLc +a064Ux5N7eOIh9gxvpDKBeSFD0eNB8IDtPQhUspr+WnoycawrJHNGawL8WIfrWY3 +0ZPF981iPCWCdN3woDP9wHA2QtBClAk2pQ1aMgdkK9r/QLO+DY92xmT/Uu4ik2jR +zv+QplsQLftjD+bRar5R9jiCWV5phPqrOF3ypMiU0K5bsnrZfGBzBcoEyfKuB+UR +F/j/631OC6yLRasr+xcL1gc+SCryfA== +=tkZT -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index b5cac23e6..8ff7944b5 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1196,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 2e52f03fdf9a1fbe2049d89f071a583bcf919976..8dd6837754301180039bdcd2192ec2ffa9911f1e 100644 GIT binary patch literal 256 zcmV+b0ssEOC2SB&#jbL|t)G85E<)sA{+|s~F5m$!7y+-fA{(45p3@Fk=k3Xi+RklE z=idnaoUY1-Gy6T(2&LYe?9T(@a*CNN5}h_$-t&E@K&w437B8*P8QflhULg&MO?mxD$RuiY1;Ik~L_O?oC&* zWM|7qv=mZh!YK!wmdDwpz@wj96nhJniJ_laAj^74?$6FZ`p5@HrirST)N_#L(FvXz GZCemMz Date: Wed, 22 Nov 2017 18:01:56 -0800 Subject: [PATCH 32/98] Add expiration date to skipped message --- certbot/renewal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/certbot/renewal.py b/certbot/renewal.py index 7d0240f73..024a815cc 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -425,7 +425,10 @@ def handle_renewal_request(config): main.renew_cert(lineage_config, plugins, renewal_candidate) renew_successes.append(renewal_candidate.fullchain) else: - renew_skipped.append(renewal_candidate.fullchain) + expiry = crypto_util.notAfter(renewal_candidate.version( + "cert", renewal_candidate.latest_common_version())) + renew_skipped.append("%s expires on %s" % (renewal_candidate.fullchain, + expiry.strftime("%Y-%m-%d"))) except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. logger.warning("Attempting to renew cert (%s) from %s produced an " From 45613fd31c3649ac5fb08a52153a8893f394af0b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 26 Jan 2018 16:02:19 -0800 Subject: [PATCH 33/98] update changelog for 0.21.1 (#5504) --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6f7be9fe..1369b0907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.21.1 - 2018-01-25 + +### Fixed + +* When creating an HTTP to HTTPS redirect in Nginx, we now ensure the Host + header of the request is set to an expected value before redirecting users to + the domain found in the header. The previous way Certbot configured Nginx + redirects was a potential security issue which you can read more about at + https://community.letsencrypt.org/t/security-issue-with-redirects-added-by-certbots-nginx-plugin/51493. +* Fixed a problem where Certbot's Apache plugin could fail HTTP-01 challenges + if basic authentication is configured for the domain you request a + certificate for. +* certbot-auto --no-bootstrap now properly tries to use Python 3.4 on RHEL 6 + based systems rather than Python 2.6. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/49?closed=1 + ## 0.21.0 - 2018-01-17 ### Added From 72b63ca5ac19cc8da0b1aba3a55dea85bca7a0eb Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 1 Feb 2018 13:14:43 -0800 Subject: [PATCH 34/98] Use "certificate" instead of "cert" in docs. --- docs/install.rst | 2 +- docs/using.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index a914586ff..c18c3cdbc 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -116,7 +116,7 @@ certbot-auto_ method, which enables you to use installer plugins that cover both of those hard topics. If you're still not convinced and have decided to use this method, -from the server that the domain you're requesting a cert for resolves +from the server that the domain you're requesting a certficate for resolves to, `install Docker`_, then issue the following command: .. code-block:: shell diff --git a/docs/using.rst b/docs/using.rst index ab4670052..e8f84e2d7 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -557,8 +557,8 @@ apologize for any inconvenience you encounter in integrating these commands into your individual environment. .. note:: ``certbot renew`` exit status will only be 1 if a renewal attempt failed. - This means ``certbot renew`` exit status will be 0 if no cert needs to be updated. - If you write a custom script and expect to run a command only after a cert was actually renewed + This means ``certbot renew`` exit status will be 0 if no certificate needs to be updated. + If you write a custom script and expect to run a command only after a certificate was actually renewed you will need to use the ``--post-hook`` since the exit status will be 0 both on successful renewal and when renewal is not necessary. From e085ff06a12a0be6033f38f092d42253c7b21515 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Mon, 5 Feb 2018 16:27:21 -0800 Subject: [PATCH 35/98] Update old issue link to point to letsencrypt community forums. (#5538) --- certbot/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index b735421f5..bc25da549 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -556,11 +556,11 @@ class Client(object): self.installer.rollback_checkpoints() self.installer.restart() except: - # TODO: suggest letshelp-letsencrypt here reporter.add_message( "An error occurred and we failed to restore your config and " - "restart your server. Please submit a bug report to " - "https://github.com/letsencrypt/letsencrypt", + "restart your server. Please post to " + "https://community.letsencrypt.org/c/server-config " + "with details about your configuration and this error you received.", reporter.HIGH_PRIORITY) raise reporter.add_message(success_msg, reporter.HIGH_PRIORITY) From 9baf75d6c887a9b69bd92c7bffe585f091c9b08e Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 6 Feb 2018 16:45:33 -0800 Subject: [PATCH 36/98] client.py changes for ACMEv2 (#5287) * Implement ACMEv2 signing of POST bodies. * Add account, and make acme_version explicit. * Remove separate NewAccount. * Rename to add v2. * Add terms_of_service_agreed. * Split out wrap_in_jws_v2 test. * Re-add too-many-public-methods. * Split Client into ClientBase / Client / ClientV2 * Use camelCase for newAccount. * Make acme_version optional parameter on .post(). This allows us to instantiate a ClientNetwork before knowing the version. * Add kid unconditionally. --- acme/acme/client.py | 324 ++++++++++++++++++++++++--------------- acme/acme/client_test.py | 74 ++++++--- acme/acme/messages.py | 1 + 3 files changed, 258 insertions(+), 141 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index dc5efbe86..5d1add5a3 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -39,39 +39,24 @@ DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class Client(object): # pylint: disable=too-many-instance-attributes - """ACME client. - - .. todo:: - Clean up raised error types hierarchy, document, and handle (wrap) - instances of `.DeserializationError` raised in `from_json()`. +class ClientBase(object): # pylint: disable=too-many-instance-attributes + """ACME client base object. :ivar messages.Directory directory: - :ivar key: `.JWK` (private) - :ivar alg: `.JWASignature` - :ivar bool verify_ssl: Verify SSL certificates? - :ivar .ClientNetwork net: Client network. Useful for testing. If not - supplied, it will be initialized using `key`, `alg` and - `verify_ssl`. - + :ivar .ClientNetwork net: Client network. + :ivar int acme_version: ACME protocol version. 1 or 2. """ - def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, - net=None): + def __init__(self, directory, net, acme_version): """Initialize. - :param directory: Directory Resource (`.messages.Directory`) or - URI from which the resource will be downloaded. - + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. + :param int acme_version: ACME protocol version. 1 or 2. """ - self.key = key - self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net - - if isinstance(directory, six.string_types): - self.directory = messages.Directory.from_json( - self.net.get(directory).json()) - else: - self.directory = directory + self.directory = directory + self.net = net + self.acme_version = acme_version @classmethod def _regr_from_response(cls, response, uri=None, terms_of_service=None): @@ -83,28 +68,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes uri=response.headers.get('Location', uri), terms_of_service=terms_of_service) - def register(self, new_reg=None): - """Register. - - :param .NewRegistration new_reg: - - :returns: Registration Resource. - :rtype: `.RegistrationResource` - - """ - new_reg = messages.NewRegistration() if new_reg is None else new_reg - assert isinstance(new_reg, messages.NewRegistration) - - response = self.net.post(self.directory[new_reg], new_reg) - # TODO: handle errors - assert response.status_code == http_client.CREATED - - # "Instance of 'Field' has no key/contact member" bug: - # pylint: disable=no-member - return self._regr_from_response(response) - def _send_recv_regr(self, regr, body): - response = self.net.post(regr.uri, body) + response = self.net.post(regr.uri, body, acme_version=self.acme_version) # TODO: Boulder returns httplib.ACCEPTED #assert response.status_code == httplib.OK @@ -153,21 +118,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ return self._send_recv_regr(regr, messages.UpdateRegistration()) - def agree_to_tos(self, regr): - """Agree to the terms-of-service. - - Agree to the terms-of-service in a Registration Resource. - - :param regr: Registration Resource. - :type regr: `.RegistrationResource` - - :returns: Updated Registration Resource. - :rtype: `.RegistrationResource` - - """ - return self.update_registration( - regr.update(body=regr.body.update(agreement=regr.terms_of_service))) - def _authzr_from_response(self, response, identifier, uri=None): authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), @@ -176,42 +126,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes raise errors.UnexpectedUpdate(authzr) return authzr - def request_challenges(self, identifier, new_authzr_uri=None): - """Request challenges. - - :param .messages.Identifier identifier: Identifier to be challenged. - :param str new_authzr_uri: Deprecated. Do not use. - - :returns: Authorization Resource. - :rtype: `.AuthorizationResource` - - """ - if new_authzr_uri is not None: - logger.debug("request_challenges with new_authzr_uri deprecated.") - new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz, new_authz) - # TODO: handle errors - assert response.status_code == http_client.CREATED - return self._authzr_from_response(response, identifier) - - def request_domain_challenges(self, domain, new_authzr_uri=None): - """Request challenges for domain names. - - This is simply a convenience function that wraps around - `request_challenges`, but works with domain names instead of - generic identifiers. See ``request_challenges`` for more - documentation. - - :param str domain: Domain name to be challenged. - :param str new_authzr_uri: Deprecated. Do not use. - - :returns: Authorization Resource. - :rtype: `.AuthorizationResource` - - """ - return self.request_challenges(messages.Identifier( - typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) - def answer_challenge(self, challb, response): """Answer challenge. @@ -227,7 +141,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self.net.post(challb.uri, response) + response = self.net.post(challb.uri, response, + acme_version=self.acme_version) try: authzr_uri = response.links['up']['url'] except KeyError: @@ -288,6 +203,133 @@ class Client(object): # pylint: disable=too-many-instance-attributes response, authzr.body.identifier, authzr.uri) return updated_authzr, response + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + response = self.net.post(self.directory[messages.Revocation], + messages.Revocation( + certificate=cert, + reason=rsn), + content_type=None, + acme_version=self.acme_version) + if response.status_code != http_client.OK: + raise errors.ClientError( + 'Successful revocation must return HTTP OK status') + +class Client(ClientBase): + """ACME client for a v1 API. + + .. todo:: + Clean up raised error types hierarchy, document, and handle (wrap) + instances of `.DeserializationError` raised in `from_json()`. + + :ivar messages.Directory directory: + :ivar key: `.JWK` (private) + :ivar alg: `.JWASignature` + :ivar bool verify_ssl: Verify SSL certificates? + :ivar .ClientNetwork net: Client network. Useful for testing. If not + supplied, it will be initialized using `key`, `alg` and + `verify_ssl`. + + """ + + def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, + net=None): + """Initialize. + + :param directory: Directory Resource (`.messages.Directory`) or + URI from which the resource will be downloaded. + + """ + # pylint: disable=too-many-arguments + self.key = key + self.net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) if net is None else net + + if isinstance(directory, six.string_types): + directory = messages.Directory.from_json( + self.net.get(directory).json()) + super(Client, self).__init__(directory=directory, + net=net, acme_version=1) + + def register(self, new_reg=None): + """Register. + + :param .NewRegistration new_reg: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` + + """ + new_reg = messages.NewRegistration() if new_reg is None else new_reg + response = self.net.post(self.directory[new_reg], new_reg, + acme_version=1) + # TODO: handle errors + assert response.status_code == http_client.CREATED + + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + return self._regr_from_response(response) + + def agree_to_tos(self, regr): + """Agree to the terms-of-service. + + Agree to the terms-of-service in a Registration Resource. + + :param regr: Registration Resource. + :type regr: `.RegistrationResource` + + :returns: Updated Registration Resource. + :rtype: `.RegistrationResource` + + """ + return self.update_registration( + regr.update(body=regr.body.update(agreement=regr.terms_of_service))) + + def request_challenges(self, identifier, new_authzr_uri=None): + """Request challenges. + + :param .messages.Identifier identifier: Identifier to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + """ + if new_authzr_uri is not None: + logger.debug("request_challenges with new_authzr_uri deprecated.") + new_authz = messages.NewAuthorization(identifier=identifier) + response = self.net.post(self.directory.new_authz, new_authz, + acme_version=1) + # TODO: handle errors + assert response.status_code == http_client.CREATED + return self._authzr_from_response(response, identifier) + + def request_domain_challenges(self, domain, new_authzr_uri=None): + """Request challenges for domain names. + + This is simply a convenience function that wraps around + `request_challenges`, but works with domain names instead of + generic identifiers. See ``request_challenges`` for more + documentation. + + :param str domain: Domain name to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + """ + return self.request_challenges(messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) + def request_issuance(self, csr, authzrs): """Request issuance. @@ -311,7 +353,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes self.directory.new_cert, req, content_type=content_type, - headers={'Accept': content_type}) + headers={'Accept': content_type}, + acme_version=1) cert_chain_uri = response.links.get('up', {}).get('url') @@ -481,37 +524,65 @@ class Client(object): # pylint: disable=too-many-instance-attributes "Recursion limit reached. Didn't get {0}".format(uri)) return chain - def revoke(self, cert, rsn): - """Revoke certificate. - :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in - `.ComparableX509` - :param int rsn: Reason code for certificate revocation. +class ClientV2(ClientBase): + """ACME client for a v2 API. - :raises .ClientError: If revocation is unsuccessful. + :ivar messages.Directory directory: + :ivar .ClientNetwork net: Client network. + """ + + def __init__(self, directory, net): + """Initialize. + + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. + """ + super(ClientV2, self).__init__(directory=directory, + net=net, acme_version=2) + + def new_account(self, new_account): + """Register. + + :param .NewRegistration new_account: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` """ - response = self.net.post(self.directory[messages.Revocation], - messages.Revocation( - certificate=cert, - reason=rsn), - content_type=None) - if response.status_code != http_client.OK: - raise errors.ClientError( - 'Successful revocation must return HTTP OK status') + response = self.net.post(self.directory['newAccount'], new_account, + acme_version=2) + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + return self._regr_from_response(response) class ClientNetwork(object): # pylint: disable=too-many-instance-attributes - """Client network.""" + """Wrapper around requests that signs POSTs for authentication. + + Also adds user agent, and handles Content-Type. + """ JSON_CONTENT_TYPE = 'application/json' JOSE_CONTENT_TYPE = 'application/jose+json' JSON_ERROR_CONTENT_TYPE = 'application/problem+json' REPLAY_NONCE_HEADER = 'Replay-Nonce' - def __init__(self, key, alg=jose.RS256, verify_ssl=True, + """Initialize. + + :param key: Account private key + :param messages.Registration account: Account object. Required if you are + planning to use .post() with acme_version=2. + :param josepy.JWASignature alg: Algoritm to use in signing JWS. + :param bool verify_ssl: Whether to verify certificates on SSL connections. + :param str user_agent: String to send as User-Agent header. + :param float timeout: Timeout for requests. + """ + def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True, user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT): + # pylint: disable=too-many-arguments self.key = key + self.account = account self.alg = alg self.verify_ssl = verify_ssl self._nonces = set() @@ -527,21 +598,29 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes except Exception: # pylint: disable=broad-except pass - def _wrap_in_jws(self, obj, nonce): + def _wrap_in_jws(self, obj, nonce, url, acme_version): """Wrap `JSONDeSerializable` object in JWS. .. todo:: Implement ``acmePath``. :param .JSONDeSerializable obj: + :param str url: The URL to which this object will be POSTed :param bytes nonce: :rtype: `.JWS` """ jobj = obj.json_dumps(indent=2).encode() logger.debug('JWS payload:\n%s', jobj) - return jws.JWS.sign( - payload=jobj, key=self.key, alg=self.alg, - nonce=nonce).json_dumps(indent=2) + kwargs = { + "alg": self.alg, + "nonce": nonce + } + if acme_version == 2: + kwargs["url"] = url + kwargs["kid"] = self.account["uri"] + kwargs["key"] = self.key + # pylint: disable=star-args + return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2) @classmethod def _check_response(cls, response, content_type=None): @@ -714,8 +793,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes else: raise - def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs): - data = self._wrap_in_jws(obj, self._get_nonce(url)) + def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, + acme_version=1, **kwargs): + data = self._wrap_in_jws(obj, self._get_nonce(url), url, acme_version) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) self._add_nonce(response) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 84620fc99..662c32942 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -104,6 +104,23 @@ class ClientTest(unittest.TestCase): self.assertEqual(self.regr, self.client.register(self.new_reg)) # TODO: test POST call arguments + def test_new_account_v2(self): + directory = messages.Directory({ + "newAccount": 'https://www.letsencrypt-demo.org/acme/new-account', + }) + from acme.client import ClientV2 + client = ClientV2(directory, self.net) + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.regr = messages.RegistrationResource( + body=messages.Registration( + contact=self.contact, key=KEY.public_key()), + uri='https://www.letsencrypt-demo.org/acme/reg/1') + + self.assertEqual(self.regr, client.new_account(self.regr)) + def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: # pylint: disable=no-member @@ -142,20 +159,23 @@ class ClientTest(unittest.TestCase): self.client.request_challenges(self.identifier) self.net.post.assert_called_once_with( self.directory.new_authz, - messages.NewAuthorization(identifier=self.identifier)) + messages.NewAuthorization(identifier=self.identifier), + acme_version=1) def test_request_challenges_deprecated_arg(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier, new_authzr_uri="hi") self.net.post.assert_called_once_with( self.directory.new_authz, - messages.NewAuthorization(identifier=self.identifier)) + messages.NewAuthorization(identifier=self.identifier), + acme_version=1) def test_request_challenges_custom_uri(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier) self.net.post.assert_called_once_with( - 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY) + 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY, + acme_version=1) def test_request_challenges_unexpected_update(self): self._prepare_response_for_request_challenges() @@ -417,7 +437,8 @@ class ClientTest(unittest.TestCase): def test_revoke(self): self.client.revoke(self.certr.body, self.rsn) self.net.post.assert_called_once_with( - self.directory[messages.Revocation], mock.ANY, content_type=None) + self.directory[messages.Revocation], mock.ANY, content_type=None, + acme_version=1) def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) @@ -432,9 +453,22 @@ class ClientTest(unittest.TestCase): self.certr, self.rsn) +class MockJSONDeSerializable(jose.JSONDeSerializable): + # pylint: disable=missing-docstring + def __init__(self, value): + self.value = value + + def to_partial_json(self): + return {'foo': self.value} + + @classmethod + def from_json(cls, value): + pass # pragma: no cover + class ClientNetworkTest(unittest.TestCase): """Tests for acme.client.ClientNetwork.""" + # pylint: disable=too-many-public-methods def setUp(self): self.verify_ssl = mock.MagicMock() @@ -453,25 +487,27 @@ class ClientNetworkTest(unittest.TestCase): self.assertTrue(self.net.verify_ssl is self.verify_ssl) def test_wrap_in_jws(self): - class MockJSONDeSerializable(jose.JSONDeSerializable): - # pylint: disable=missing-docstring - def __init__(self, value): - self.value = value - - def to_partial_json(self): - return {'foo': self.value} - - @classmethod - def from_json(cls, value): - pass # pragma: no cover - # pylint: disable=protected-access jws_dump = self.net._wrap_in_jws( - MockJSONDeSerializable('foo'), nonce=b'Tg') + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url", + acme_version=1) jws = acme_jws.JWS.json_loads(jws_dump) self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) self.assertEqual(jws.signature.combined.nonce, b'Tg') + def test_wrap_in_jws_v2(self): + self.net.account = {'uri': 'acct-uri'} + # pylint: disable=protected-access + jws_dump = self.net._wrap_in_jws( + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url", + acme_version=2) + jws = acme_jws.JWS.json_loads(jws_dump) + self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) + self.assertEqual(jws.signature.combined.nonce, b'Tg') + self.assertEqual(jws.signature.combined.kid, u'acct-uri') + self.assertEqual(jws.signature.combined.url, u'url') + + def test_check_response_not_ok_jobj_no_error(self): self.response.ok = False self.response.json.return_value = {} @@ -701,13 +737,13 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.assertEqual(self.checked_response, self.net.post( 'uri', self.obj, content_type=self.content_type)) self.net._wrap_in_jws.assert_called_once_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) self.available_nonces = [] self.assertRaises(errors.MissingNonce, self.net.post, 'uri', self.obj, content_type=self.content_type) self.net._wrap_in_jws.assert_called_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) def test_post_wrong_initial_nonce(self): # HEAD self.available_nonces = [b'f', jose.b64encode(b'good')] diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 6daf55094..98993c4e1 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -251,6 +251,7 @@ class Registration(ResourceBody): contact = jose.Field('contact', omitempty=True, default=()) agreement = jose.Field('agreement', omitempty=True) status = jose.Field('status', omitempty=True) + terms_of_service_agreed = jose.Field('terms-of-service-agreed', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' From 0416382633c7b37f05193c0bc42d10d03ac13b5e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Feb 2018 17:01:58 -0800 Subject: [PATCH 37/98] Update leauto_upgrades with tests from #5402. (#5407) --- .../letstest/scripts/test_leauto_upgrades.sh | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 51472f2e6..355fead2e 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -65,6 +65,7 @@ iQIDAQAB " if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then + RUN_PYTHON3_TESTS=1 if command -v python3; then echo "Didn't expect Python 3 to be installed!" exit 1 @@ -85,11 +86,25 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; exit 1 fi unset VENV_PATH - EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) - if ! ./cb-auto -v --debug --version -n 2>&1 | grep "$EXPECTED_VERSION" ; then - echo "Certbot didn't upgrade as expected!" - exit 1 - fi +fi + +if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then + echo "Had problems checking for updates!" + exit 1 +fi + +EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) +if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | grep "$EXPECTED_VERSION" ; then + echo upgrade appeared to fail + exit 1 +fi + +if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then + echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ + exit 1 +fi + +if [ "$RUN_PYTHON3_TESTS" = 1 ]; then if ! command -v python3; then echo "Python3 wasn't properly installed" exit 1 @@ -98,11 +113,7 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; echo "Python3 wasn't used in venv!" exit 1 fi -elif ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then - echo upgrade appeared to fail - exit 1 fi - echo upgrade appeared to be successful if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then From 530a9590e6b9a912162facdae58924f0e750d314 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 7 Feb 2018 14:08:03 -0800 Subject: [PATCH 38/98] Add sudo to certbot-auto instructions. (#5501) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index c144a4f74..a3e5d530a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -32,7 +32,7 @@ a new plugin is introduced. .. code-block:: shell cd certbot - ./certbot-auto --os-packages-only + sudo ./certbot-auto --os-packages-only ./tools/venv.sh Then in each shell where you're working on the client, do: From 4f0aeb12fa30d56c3926713901bc02cc53d6b9d1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Feb 2018 14:14:26 -0800 Subject: [PATCH 39/98] Add find-duplicative-certs docs (#5547) * add find-duplicative-certs docs * address review feedback --- certbot/cert_manager.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index da85ed783..4240a0523 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -121,7 +121,28 @@ def domains_for_certname(config, certname): return lineage.names() if lineage else None def find_duplicative_certs(config, domains): - """Find existing certs that duplicate the request.""" + """Find existing certs that match the given domain names. + + This function searches for certificates whose domains are equal to + the `domains` parameter and certificates whose domains are a subset + of the domains in the `domains` parameter. If multiple certificates + are found whose names are a subset of `domains`, the one whose names + are the largest subset of `domains` is returned. + + If multiple certificates' domains are an exact match or equally + sized subsets, which matching certificates are returned is + undefined. + + :param config: Configuration. + :type config: :class:`certbot.configuration.NamespaceConfig` + :param domains: List of domain names + :type domains: `list` of `str` + + :returns: lineages representing the identically matching cert and the + largest subset if they exist + :rtype: `tuple` of `storage.RenewableCert` or `None` + + """ def update_certs_for_domain_matches(candidate_lineage, rv): """Return cert as identical_names_cert if it matches, or subset_names_cert if it matches as subset From d6b247c002d9e5986098c44c771f2177a0823aae Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 9 Feb 2018 12:54:15 -0800 Subject: [PATCH 40/98] Set ClientNetwork.account after registering (#5558) --- acme/acme/client.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 5d1add5a3..77e89e535 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -95,6 +95,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes update = regr.body if update is None else update body = messages.UpdateRegistration(**dict(update)) updated_regr = self._send_recv_regr(regr, body=body) + self.net.account = updated_regr return updated_regr def deactivate_registration(self, regr): @@ -555,8 +556,9 @@ class ClientV2(ClientBase): acme_version=2) # "Instance of 'Field' has no key/contact member" bug: # pylint: disable=no-member - return self._regr_from_response(response) - + regr = self._regr_from_response(response) + self.net.account = regr + return regr class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Wrapper around requests that signs POSTs for authentication. @@ -571,8 +573,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Initialize. :param key: Account private key - :param messages.Registration account: Account object. Required if you are - planning to use .post() with acme_version=2. + :param messages.RegistrationResource account: Account object. Required if you are + planning to use .post() with acme_version=2 for anything other than creating a new + account; may be set later after registering. :param josepy.JWASignature alg: Algoritm to use in signing JWS. :param bool verify_ssl: Whether to verify certificates on SSL connections. :param str user_agent: String to send as User-Agent header. From 1f45832460550d0982676368870af9cac69592a5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 9 Feb 2018 16:41:05 -0800 Subject: [PATCH 41/98] Suggest people try the community forum. (#5561) --- ISSUE_TEMPLATE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 55780c0ac..03917f8ca 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,3 +1,9 @@ +If you're having trouble using Certbot and aren't sure you've found a bug or +request for a new feature, please first try asking for help at +https://community.letsencrypt.org/. There is a much larger community there of +people familiar with the project who will be able to more quickly answer your +questions. + ## My operating system is (include version): From abc4a27613f1e8196a52f75b051c05c3d2153a2a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 12 Feb 2018 14:07:33 -0800 Subject: [PATCH 42/98] [Docs] restore docs for ppl just using Certbot git master (#5420) - Dev / test cycles are one use case for the "running a local copy of the client" instructions, but simply running bleeding edge Certbot is another - So edit the docs to once again explain how to just run bleeding edge Certbot, without (say) always getting staging certs. --- docs/contributing.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index a3e5d530a..83b607e15 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -35,7 +35,10 @@ a new plugin is introduced. sudo ./certbot-auto --os-packages-only ./tools/venv.sh -Then in each shell where you're working on the client, do: +You can now run the copy of Certbot from git either by executing +``venv/bin/certbot``, or by activating the virtual environment. If you're +actively modifying and testing the code, you may want to run commands like this in +each shell where you're working: .. code-block:: shell @@ -43,7 +46,8 @@ Then in each shell where you're working on the client, do: export SERVER=https://acme-staging.api.letsencrypt.org/directory source tests/integration/_common.sh -After that, your shell will be using the virtual environment, and you run the +After that, your shell will be using the virtual environment, your copy of +Certbot will default to requesting test (staging) certificates, and you run the client by typing `certbot` or `certbot_test`. The latter is an alias that includes several flags useful for testing. For instance, it sets various output directories to point to /tmp/, and uses non-privileged ports for challenges, so From 789be8f9bc36241eee88849a6c496dc898185030 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 12 Feb 2018 14:55:41 -0800 Subject: [PATCH 43/98] Change "Attempting to parse" warning to info. (#5557) * Change "Attempting to parse" warning to info. This message shows up on every renewal run when the config was updated by a newer version of Certbot than the one being run. For instance, if a user has the certbot packages installed from PPA (currently 0.18.2), but runs certbot-auto once to try out the latest version (0.21.1), they will start getting this message via email every 12 hours. --- certbot/storage.py | 2 +- certbot/tests/storage_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot/storage.py b/certbot/storage.py index 67d2155ae..ed3922c58 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -417,7 +417,7 @@ class RenewableCert(object): conf_version = self.configuration.get("version") if (conf_version is not None and util.get_strict_version(conf_version) > CURRENT_VERSION): - logger.warning( + logger.info( "Attempting to parse the version %s renewal configuration " "file found at %s with version %s of Certbot. This might not " "work.", conf_version, config_filename, certbot.__version__) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 6c8f775e2..6c0970e72 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -158,8 +158,8 @@ class RenewableCertTests(BaseRenewableCertTest): with mock.patch("certbot.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) - self.assertTrue(mock_logger.warning.called) - self.assertTrue("version" in mock_logger.warning.call_args[0][0]) + self.assertTrue(mock_logger.info.called) + self.assertTrue("version" in mock_logger.info.call_args[0][0]) def test_consistent(self): # pylint: disable=too-many-statements,protected-access From 90664f196f992b20bdfdfb8914697e4456de7510 Mon Sep 17 00:00:00 2001 From: Eli Young Date: Mon, 12 Feb 2018 16:43:11 -0800 Subject: [PATCH 44/98] Remove autodocs for long-removed acme.other module (#5529) This module was removed in 22a9c7e3c2a811698ed7d63fae1cd43d0bd5d088. The autodocs are therefore unnecessary. Furthermore, they are starting to cause build failures for Fedora. --- acme/docs/api/other.rst | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 acme/docs/api/other.rst diff --git a/acme/docs/api/other.rst b/acme/docs/api/other.rst deleted file mode 100644 index eb27a5d53..000000000 --- a/acme/docs/api/other.rst +++ /dev/null @@ -1,5 +0,0 @@ -Other ACME objects ------------------- - -.. automodule:: acme.other - :members: From 932ecbb9c20bdc4e9d05c2e2bf78ea1b1608b67d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 13 Feb 2018 02:43:59 +0200 Subject: [PATCH 45/98] Fix test inconsistence in Apache plugin configurator_test (#5520) --- certbot-apache/certbot_apache/tests/configurator_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 530d75a92..7fbfcb617 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -474,7 +474,11 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(mock_add_dir.call_count, 3) self.assertTrue(mock_add_dir.called) self.assertEqual(mock_add_dir.call_args[0][1], "Listen") - self.assertEqual(mock_add_dir.call_args[0][2], ['1.2.3.4:8080']) + call_found = False + for mock_call in mock_add_dir.mock_calls: + if mock_call[1][2] == ['1.2.3.4:8080']: + call_found = True + self.assertTrue(call_found) def test_prepare_server_https(self): mock_enable = mock.Mock() From 49edf17cb77455f9f7b8227735939f693ceff303 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 13 Feb 2018 09:52:04 -0800 Subject: [PATCH 46/98] ignore .docker (#5477) --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index a01d2e1c7..4dff20caf 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ tests/letstest/venv/ # pytest cache .cache + +# docker files +.docker From ad0a99a1f5aa3cc28988685428bfdd475d48df58 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Tue, 13 Feb 2018 10:50:04 -0800 Subject: [PATCH 47/98] Proper webroot directory cleanup (#5453) * fix(webroot): clean up directories properly * fix(webroot): undo umask in finally --- certbot/plugins/util.py | 18 ++++++++++ certbot/plugins/util_test.py | 8 +++++ certbot/plugins/webroot.py | 63 +++++++++++++++++---------------- certbot/plugins/webroot_test.py | 31 ++++++++++++++++ 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index f0e2f4c5b..ad2257e1d 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,6 +6,24 @@ from certbot import util logger = logging.getLogger(__name__) +def get_prefixes(path): + """Retrieves all possible path prefixes of a path, in descending order + of length. For instance, + /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + :param str path: the path to break into prefixes + + :returns: all possible path prefixes of given path in descending order + :rtype: `list` of `str` + """ + prefix = path + prefixes = [] + while len(prefix) > 0: + prefixes.append(prefix) + prefix, _ = os.path.split(prefix) + # break once we hit '/' + if prefix == prefixes[-1]: + break + return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 947f24697..2c0e476ae 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,6 +5,14 @@ import unittest import mock +class GetPrefixTest(unittest.TestCase): + """Tests for certbot.plugins.get_prefixes.""" + def test_get_prefix(self): + from certbot.plugins.util import get_prefixes + self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes("/"), ["/"]) + self.assertEqual(get_prefixes("a"), ["a"]) + class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 714d83cce..15889a25c 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,6 +18,7 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common +from certbot.plugins import util logger = logging.getLogger(__name__) @@ -65,6 +66,8 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) + # stack of dirs successfully created by this authenticator + self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -161,27 +164,26 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) - try: - # This is coupled with the "umask" call above because - # os.makedirs's "mode" parameter may not always work: - # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python - os.makedirs(self.full_roots[name], 0o0755) - - # Set owner as parent directory if possible - try: - stat_path = os.stat(path) - os.chown(self.full_roots[name], stat_path.st_uid, - stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + stat_path = os.stat(path) + for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + try: + # This is coupled with the "umask" call above because + # os.mkdir's "mode" parameter may not always work: + # https://docs.python.org/3/library/os.html#os.mkdir + os.mkdir(prefix, 0o0755) + self._created_dirs.append(prefix) + # Set owner as parent directory if possible + try: + os.chown(prefix, stat_path.st_uid, stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}", name, exception) finally: os.umask(old_umask) @@ -217,16 +219,17 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - for root_path, achalls in six.iteritems(self.performed): - if not achalls: - try: - os.rmdir(root_path) - logger.debug("All challenges cleaned up, removing %s", - root_path) - except OSError as exc: - logger.info( - "Unable to clean up challenge directory %s", root_path) - logger.debug("Error was: %s", exc) + not_removed = [] + while len(self._created_dirs) > 0: + path = self._created_dirs.pop() + try: + os.rmdir(path) + except OSError as exc: + not_removed.insert(0, path) + logger.info("Challenge directory %s was not empty, didn't remove", path) + logger.debug("Error was: %s", exc) + self._created_dirs = not_removed + logger.debug("All challenges cleaned up") class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 36e2ffba6..59133f0aa 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,6 +36,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() + self.partial_root_challenge_path = os.path.join( + self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -199,6 +201,35 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) + self.assertFalse(os.path.exists(self.partial_root_challenge_path)) + + def test_perform_cleanup_existing_dirs(self): + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([self.achall]) + self.auth.cleanup([self.achall]) + + # Ensure we don't "clean up" directories that previously existed + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) + + def test_perform_cleanup_multiple_challenges(self): + bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"bingo"), "pending"), + domain="thing.com", account_key=KEY) + + bingo_validation_path = "YmluZ28" + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([bingo_achall, self.achall]) + + self.auth.cleanup([self.achall]) + self.assertFalse(os.path.exists(bingo_validation_path)) + self.assertTrue(os.path.exists(self.root_challenge_path)) + self.auth.cleanup([bingo_achall]) + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() From 9277710f6f984f6464ee375ea72d3a11b689853c Mon Sep 17 00:00:00 2001 From: sydneyli Date: Tue, 13 Feb 2018 11:15:08 -0800 Subject: [PATCH 48/98] Added install-only flag (#5531) --- letsencrypt-auto-source/letsencrypt-auto | 9 +++++++++ letsencrypt-auto-source/letsencrypt-auto.template | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8ff7944b5..c84499eab 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -1428,6 +1431,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2ce337002..365fc6752 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -587,6 +590,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else From ac464a58e5591f917ab66e2c30b2ba92cdc85e80 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 14 Feb 2018 18:16:20 +0200 Subject: [PATCH 49/98] Only add Include for TLS configuration if not already there (#5498) * Only add Include for TLS configuration if not already there * Add tests to prevent future regression --- certbot-apache/certbot_apache/configurator.py | 5 +++- .../certbot_apache/tests/configurator_test.py | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index b32eda921..4bb2cbebd 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1269,7 +1269,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "insert_cert_file_path") self.parser.add_dir(vh_path, "SSLCertificateKeyFile", "insert_key_file_path") - self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) + # Only include the TLS configuration if not already included + existing_inc = self.parser.find_dir("Include", self.mod_ssl_conf, vh_path) + if not existing_inc: + self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) def _add_servername_alias(self, target_name, vhost): vh_path = vhost.path diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 7fbfcb617..8f34d33d3 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -335,6 +335,33 @@ class MultipleVhostsTest(util.ApacheTest): "example/cert_chain.pem", "example/fullchain.pem") self.assertTrue(ssl_vhost.enabled) + def test_no_duplicate_include(self): + def mock_find_dir(directive, argument, _): + """Mock method for parser.find_dir""" + if directive == "Include" and argument.endswith("options-ssl-apache.conf"): + return ["/path/to/whatever"] + + mock_add = mock.MagicMock() + self.config.parser.add_dir = mock_add + self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access + tried_to_add = False + for a in mock_add.call_args_list: + if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf: + tried_to_add = True + # Include should be added, find_dir is not patched, and returns falsy + self.assertTrue(tried_to_add) + + self.config.parser.find_dir = mock_find_dir + mock_add.reset_mock() + + self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access + tried_to_add = [] + for a in mock_add.call_args_list: + tried_to_add.append(a[0][1] == "Include" and + a[0][2] == self.config.mod_ssl_conf) + # Include shouldn't be added, as patched find_dir "finds" existing one + self.assertFalse(any(tried_to_add)) + def test_deploy_cert(self): self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") From fbace69b5e8fed5ee32cb029478d6b11aea84842 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 14 Feb 2018 19:28:36 +0200 Subject: [PATCH 50/98] Fix install verb (#5536) * Fix install verb * Fix error message, tests and remove global pylint change * Fix boulder integration test keypath * Also use chain_path from lineage if not defined on CLI --- certbot/cli.py | 5 +- certbot/main.py | 41 ++++++++++++++-- certbot/tests/main_test.py | 90 +++++++++++++++++++++++++++++++++--- tests/boulder-integration.sh | 2 +- 4 files changed, 125 insertions(+), 13 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index dec7474f9..09dd71d13 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1278,14 +1278,13 @@ def _paths_parser(helpful): elif verb == "revoke": add(section, "--cert-path", type=read_file, required=True, help=cph) else: - add(section, "--cert-path", type=os.path.abspath, - help=cph, required=(verb == "install")) + add(section, "--cert-path", type=os.path.abspath, help=cph) section = "paths" if verb in ("install", "revoke"): section = verb # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", required=(verb == "install"), + add(section, "--key-path", type=((verb == "revoke" and read_file) or os.path.abspath), help="Path to private key for certificate installation " "or revocation (if account key is missing)") diff --git a/certbot/main.py b/certbot/main.py index 32dd69256..13234e0d2 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1,4 +1,5 @@ """Certbot main entry point.""" +# pylint: disable=too-many-lines from __future__ import print_function import functools import logging.handlers @@ -779,11 +780,45 @@ def install(config, plugins): except errors.PluginSelectionError as e: return str(e) - domains, _ = _find_domains_or_certname(config, installer) - le_client = _init_le_client(config, authenticator=None, installer=installer) - _install_cert(config, le_client, domains) + # If cert-path is defined, populate missing (ie. not overridden) values. + # Unfortunately this can't be done in argument parser, as certificate + # manager needs the access to renewal directory paths + if config.certname: + config = _populate_from_certname(config) + if config.key_path and config.cert_path: + _check_certificate_and_key(config) + domains, _ = _find_domains_or_certname(config, installer) + le_client = _init_le_client(config, authenticator=None, installer=installer) + _install_cert(config, le_client, domains) + else: + raise errors.ConfigurationError("Path to certificate or key was not defined. " + "If your certificate is managed by Certbot, please use --cert-name " + "to define which certificate you would like to install.") +def _populate_from_certname(config): + """Helper function for install to populate missing config values from lineage + defined by --cert-name.""" + lineage = cert_manager.lineage_for_certname(config, config.certname) + if not lineage: + return config + if not config.key_path: + config.namespace.key_path = lineage.key_path + if not config.cert_path: + config.namespace.cert_path = lineage.cert_path + if not config.chain_path: + config.namespace.chain_path = lineage.chain_path + if not config.fullchain_path: + config.namespace.fullchain_path = lineage.fullchain_path + return config + +def _check_certificate_and_key(config): + if not os.path.isfile(os.path.realpath(config.cert_path)): + raise errors.ConfigurationError("Error while reading certificate from path " + "{0}".format(config.cert_path)) + if not os.path.isfile(os.path.realpath(config.key_path)): + raise errors.ConfigurationError("Error while reading private key from path " + "{0}".format(config.key_path)) def plugins_cmd(config, plugins): """List server software plugins. diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index b1d58542f..7368e8513 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -592,11 +592,30 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met super(MainTest, self).tearDown() - def _call(self, args, stdout=None): - "Run the cli with output streams and actual client mocked out" - with mock.patch('certbot.main.client') as client: - ret, stdout, stderr = self._call_no_clientmock(args, stdout) - return ret, stdout, stderr, client + def _call(self, args, stdout=None, mockisfile=False): + """Run the cli with output streams, actual client and optionally + os.path.isfile() mocked out""" + + if mockisfile: + orig_open = os.path.isfile + def mock_isfile(fn, *args, **kwargs): + """Mock os.path.isfile()""" + if (fn.endswith("cert") or + fn.endswith("chain") or + fn.endswith("privkey")): + return True + else: + return orig_open(fn, *args, **kwargs) + + with mock.patch("os.path.isfile") as mock_if: + mock_if.side_effect = mock_isfile + with mock.patch('certbot.main.client') as client: + ret, stdout, stderr = self._call_no_clientmock(args, stdout) + return ret, stdout, stderr, client + else: + with mock.patch('certbot.main.client') as client: + ret, stdout, stderr = self._call_no_clientmock(args, stdout) + return ret, stdout, stderr, client def _call_no_clientmock(self, args, stdout=None): "Run the client with output streams mocked out" @@ -680,9 +699,68 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_selection(self, mock_pick_installer, _rec): self._call(['install', '--domains', 'foo.bar', '--cert-path', 'cert', - '--key-path', 'key', '--chain-path', 'chain']) + '--key-path', 'privkey', '--chain-path', 'chain'], mockisfile=True) self.assertEqual(mock_pick_installer.call_count, 1) + @mock.patch('certbot.main._install_cert') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_certname(self, _inst, _rec, mock_install): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install', '--cert-name', 'whatever'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/privkey") + + @mock.patch('certbot.main._install_cert') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_param_override(self, _inst, _rec, mock_install): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install', '--cert-name', 'whatever', + '--key-path', '/tmp/overriding_privkey'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.chain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/overriding_privkey") + + mock_install.reset() + + self._call(['install', '--cert-name', 'whatever', + '--cert-path', '/tmp/overriding_cert'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/overriding_cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/privkey") + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_param_error(self, _inst, _rec): + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--key-path', '/tmp/key_path']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--cert-path', '/tmp/key_path']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--cert-name', 'notfound', + '--key-path', 'invalid']) + @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.util.exe_exists') def test_configurator_selection(self, mock_exe_exists, unused_report): diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index e1aad4336..24d224cb0 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -251,7 +251,7 @@ openssl x509 -in "${root}/csr/chain.pem" -text common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ - --key-path "${root}/csr/key.pem" + --key-path "${root}/key.pem" CheckCertCount() { CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` From 99aec1394df3813646a2734405043aeada160a49 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Wed, 14 Feb 2018 12:09:17 -0800 Subject: [PATCH 51/98] Revert "Proper webroot directory cleanup (#5453)" (#5574) This reverts commit ad0a99a1f5aa3cc28988685428bfdd475d48df58. --- certbot/plugins/util.py | 18 ---------- certbot/plugins/util_test.py | 8 ----- certbot/plugins/webroot.py | 63 ++++++++++++++++----------------- certbot/plugins/webroot_test.py | 31 ---------------- 4 files changed, 30 insertions(+), 90 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index ad2257e1d..f0e2f4c5b 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,24 +6,6 @@ from certbot import util logger = logging.getLogger(__name__) -def get_prefixes(path): - """Retrieves all possible path prefixes of a path, in descending order - of length. For instance, - /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] - :param str path: the path to break into prefixes - - :returns: all possible path prefixes of given path in descending order - :rtype: `list` of `str` - """ - prefix = path - prefixes = [] - while len(prefix) > 0: - prefixes.append(prefix) - prefix, _ = os.path.split(prefix) - # break once we hit '/' - if prefix == prefixes[-1]: - break - return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 2c0e476ae..947f24697 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,14 +5,6 @@ import unittest import mock -class GetPrefixTest(unittest.TestCase): - """Tests for certbot.plugins.get_prefixes.""" - def test_get_prefix(self): - from certbot.plugins.util import get_prefixes - self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) - self.assertEqual(get_prefixes("/"), ["/"]) - self.assertEqual(get_prefixes("a"), ["a"]) - class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 15889a25c..714d83cce 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,7 +18,6 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common -from certbot.plugins import util logger = logging.getLogger(__name__) @@ -66,8 +65,6 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) - # stack of dirs successfully created by this authenticator - self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -164,26 +161,27 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) + try: - stat_path = os.stat(path) - for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): - try: - # This is coupled with the "umask" call above because - # os.mkdir's "mode" parameter may not always work: - # https://docs.python.org/3/library/os.html#os.mkdir - os.mkdir(prefix, 0o0755) - self._created_dirs.append(prefix) - # Set owner as parent directory if possible - try: - os.chown(prefix, stat_path.st_uid, stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + # This is coupled with the "umask" call above because + # os.makedirs's "mode" parameter may not always work: + # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python + os.makedirs(self.full_roots[name], 0o0755) + + # Set owner as parent directory if possible + try: + stat_path = os.stat(path) + os.chown(self.full_roots[name], stat_path.st_uid, + stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + + except OSError as exception: + if exception.errno != errno.EEXIST: + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}", name, exception) finally: os.umask(old_umask) @@ -219,17 +217,16 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - not_removed = [] - while len(self._created_dirs) > 0: - path = self._created_dirs.pop() - try: - os.rmdir(path) - except OSError as exc: - not_removed.insert(0, path) - logger.info("Challenge directory %s was not empty, didn't remove", path) - logger.debug("Error was: %s", exc) - self._created_dirs = not_removed - logger.debug("All challenges cleaned up") + for root_path, achalls in six.iteritems(self.performed): + if not achalls: + try: + os.rmdir(root_path) + logger.debug("All challenges cleaned up, removing %s", + root_path) + except OSError as exc: + logger.info( + "Unable to clean up challenge directory %s", root_path) + logger.debug("Error was: %s", exc) class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 59133f0aa..36e2ffba6 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,8 +36,6 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() - self.partial_root_challenge_path = os.path.join( - self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -201,35 +199,6 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) - self.assertFalse(os.path.exists(self.partial_root_challenge_path)) - - def test_perform_cleanup_existing_dirs(self): - os.mkdir(self.partial_root_challenge_path) - self.auth.prepare() - self.auth.perform([self.achall]) - self.auth.cleanup([self.achall]) - - # Ensure we don't "clean up" directories that previously existed - self.assertFalse(os.path.exists(self.validation_path)) - self.assertFalse(os.path.exists(self.root_challenge_path)) - - def test_perform_cleanup_multiple_challenges(self): - bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.HTTP01(token=b"bingo"), "pending"), - domain="thing.com", account_key=KEY) - - bingo_validation_path = "YmluZ28" - os.mkdir(self.partial_root_challenge_path) - self.auth.prepare() - self.auth.perform([bingo_achall, self.achall]) - - self.auth.cleanup([self.achall]) - self.assertFalse(os.path.exists(bingo_validation_path)) - self.assertTrue(os.path.exists(self.root_challenge_path)) - self.auth.cleanup([bingo_achall]) - self.assertFalse(os.path.exists(self.validation_path)) - self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() From 608875cd655e5719a56d178e4046539c98b91348 Mon Sep 17 00:00:00 2001 From: Sydney Li Date: Wed, 14 Feb 2018 15:14:24 -0800 Subject: [PATCH 52/98] Add test for skipped certs --- certbot/tests/main_test.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 45e5db1df..8a2ffd1ca 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -926,7 +926,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, - quiet_mode=False): + quiet_mode=False, expiry_date=datetime.datetime.now()): # pylint: disable=too-many-locals,too-many-arguments cert_path = test_util.vector_path('cert_512.pem') chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' @@ -959,7 +959,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mock_latest = mock.MagicMock() mock_latest.get_issuer.return_value = "Fake fake" mock_ssl.crypto.load_certificate.return_value = mock_latest - with mock.patch('certbot.main.renewal.crypto_util'): + with mock.patch('certbot.main.renewal.crypto_util') as mock_crypto_util: + mock_crypto_util.notAfter.return_value = expiry_date if not args: args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly'] if extra_args: @@ -1027,6 +1028,16 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, should_renew=True) + @mock.patch('certbot.renewal.should_renew') + def test_renew_skips_recent_certs(self, should_renew): + should_renew.return_value = False + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + expiry = datetime.datetime.now() + datetime.timedelta(days=90) + _, _, stdout = self._test_renewal_common(False, extra_args=None, should_renew=False, + args=['renew'], expiry_date=expiry) + self.assertTrue('No renewals were attempted.' in stdout.getvalue()) + self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue()) + def test_quiet_renew(self): test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run"] From 09b5927e6ac881359f6df9a1792a598fd94ad8d6 Mon Sep 17 00:00:00 2001 From: cclauss Date: Thu, 15 Feb 2018 20:07:35 +0100 Subject: [PATCH 53/98] from botocore.exceptions import ClientError (#5507) Fixes undefined name 'botocore' in flake8 testing of https://github.com/certbot/certbot $ __flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics__ ``` ./tests/letstest/multitester.py:144:12: F821 undefined name 'botocore' except botocore.exceptions.ClientError as e: ^ 1 F821 undefined name 'botocore' ``` --- tests/letstest/multitester.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index d9491939c..17740cde8 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -38,6 +38,7 @@ from multiprocessing import Manager import urllib2 import yaml import boto3 +from botocore.exceptions import ClientError import fabric from fabric.api import run, execute, local, env, sudo, cd, lcd from fabric.operations import get, put @@ -141,7 +142,7 @@ def make_instance(instance_name, # give instance a name try: new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) - except botocore.exceptions.ClientError as e: + except ClientError as e: if "InvalidInstanceID.NotFound" in str(e): # This seems to be ephemeral... retry time.sleep(1) From d5efefd979b7c660130b04e5c588d2f4a63a991c Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 15 Feb 2018 15:55:08 -0800 Subject: [PATCH 54/98] Re-land proper webroot directory cleanup (#5577) * fix(webroot): clean up directories properly * fix(webroot): undo umask in finally * Fix for MacOS --- certbot/plugins/util.py | 18 ++++++++++ certbot/plugins/util_test.py | 8 +++++ certbot/plugins/webroot.py | 63 +++++++++++++++++---------------- certbot/plugins/webroot_test.py | 31 ++++++++++++++++ 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index f0e2f4c5b..ad2257e1d 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,6 +6,24 @@ from certbot import util logger = logging.getLogger(__name__) +def get_prefixes(path): + """Retrieves all possible path prefixes of a path, in descending order + of length. For instance, + /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + :param str path: the path to break into prefixes + + :returns: all possible path prefixes of given path in descending order + :rtype: `list` of `str` + """ + prefix = path + prefixes = [] + while len(prefix) > 0: + prefixes.append(prefix) + prefix, _ = os.path.split(prefix) + # break once we hit '/' + if prefix == prefixes[-1]: + break + return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 947f24697..2c0e476ae 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,6 +5,14 @@ import unittest import mock +class GetPrefixTest(unittest.TestCase): + """Tests for certbot.plugins.get_prefixes.""" + def test_get_prefix(self): + from certbot.plugins.util import get_prefixes + self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes("/"), ["/"]) + self.assertEqual(get_prefixes("a"), ["a"]) + class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 714d83cce..6328b16ef 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,6 +18,7 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common +from certbot.plugins import util logger = logging.getLogger(__name__) @@ -65,6 +66,8 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) + # stack of dirs successfully created by this authenticator + self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -161,27 +164,26 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) - try: - # This is coupled with the "umask" call above because - # os.makedirs's "mode" parameter may not always work: - # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python - os.makedirs(self.full_roots[name], 0o0755) - - # Set owner as parent directory if possible - try: - stat_path = os.stat(path) - os.chown(self.full_roots[name], stat_path.st_uid, - stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + stat_path = os.stat(path) + for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + try: + # This is coupled with the "umask" call above because + # os.mkdir's "mode" parameter may not always work: + # https://docs.python.org/3/library/os.html#os.mkdir + os.mkdir(prefix, 0o0755) + self._created_dirs.append(prefix) + # Set owner as parent directory if possible + try: + os.chown(prefix, stat_path.st_uid, stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + except OSError as exception: + if exception.errno not in (errno.EEXIST, errno.EISDIR): + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}".format(name, exception)) finally: os.umask(old_umask) @@ -217,16 +219,17 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - for root_path, achalls in six.iteritems(self.performed): - if not achalls: - try: - os.rmdir(root_path) - logger.debug("All challenges cleaned up, removing %s", - root_path) - except OSError as exc: - logger.info( - "Unable to clean up challenge directory %s", root_path) - logger.debug("Error was: %s", exc) + not_removed = [] + while len(self._created_dirs) > 0: + path = self._created_dirs.pop() + try: + os.rmdir(path) + except OSError as exc: + not_removed.insert(0, path) + logger.info("Challenge directory %s was not empty, didn't remove", path) + logger.debug("Error was: %s", exc) + self._created_dirs = not_removed + logger.debug("All challenges cleaned up") class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 36e2ffba6..59133f0aa 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,6 +36,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() + self.partial_root_challenge_path = os.path.join( + self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -199,6 +201,35 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) + self.assertFalse(os.path.exists(self.partial_root_challenge_path)) + + def test_perform_cleanup_existing_dirs(self): + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([self.achall]) + self.auth.cleanup([self.achall]) + + # Ensure we don't "clean up" directories that previously existed + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) + + def test_perform_cleanup_multiple_challenges(self): + bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"bingo"), "pending"), + domain="thing.com", account_key=KEY) + + bingo_validation_path = "YmluZ28" + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([bingo_achall, self.achall]) + + self.auth.cleanup([self.achall]) + self.assertFalse(os.path.exists(bingo_validation_path)) + self.assertTrue(os.path.exists(self.root_challenge_path)) + self.auth.cleanup([bingo_achall]) + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() From d467a4ae95680220286c5ce7e6e840b48c8f74a5 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 15 Feb 2018 19:04:17 -0800 Subject: [PATCH 55/98] Add mechanism to detect acme version (#5554) Detects acme version by checking for newNonce field in the directory, since it's mandatory. Also updates ClientNetwork.account on register and update_registration. * add mechanism to detect acme version * update ClientNetwork.account comment * switch to MultiVersionClient object in acme * add shim methods * add returns * use backwards-compatible format and implement register * update to actual representation of tos v2 * add tos fields and pass through to v1 for partial updates * update tests * pass more tests * allow instance variable pass-through and lint * update certbot and tests to use new_account_and_tos method * remove --agree-tos test from main_test for now because we moved the callback into acme * add docstrings * use hasattr * all most review comments * use terms_of_service for both v1 and v2 * add tests for acme/client.py * tests for acme/messages.py --- acme/acme/client.py | 56 ++++++++++++ acme/acme/client_test.py | 171 ++++++++++++++++++++++++++++------- acme/acme/messages.py | 25 ++++- acme/acme/messages_test.py | 7 ++ certbot/client.py | 24 ++--- certbot/main.py | 11 ++- certbot/tests/client_test.py | 36 ++++---- certbot/tests/main_test.py | 17 ++-- certbot/tests/util.py | 2 +- 9 files changed, 262 insertions(+), 87 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 77e89e535..d843feaa7 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -560,6 +560,62 @@ class ClientV2(ClientBase): self.net.account = regr return regr +class BackwardsCompatibleClientV2(object): + """ACME client wrapper that tends towards V2-style calls, but + supports V1 servers. + + :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint + :ivar .ClientBase client: either Client or ClientV2 + """ + + def __init__(self, net, key, server): + directory = messages.Directory.from_json(net.get(server).json()) + self.acme_version = self._acme_version_from_directory(directory) + if self.acme_version == 1: + self.client = Client(directory, key=key, net=net) + else: + self.client = ClientV2(directory, net=net) + + def __getattr__(self, name): + if name in vars(self.client): + return getattr(self.client, name) + elif name in dir(ClientBase): + return getattr(self.client, name) + # temporary, for breaking changes into smaller pieces + elif name in dir(Client): + return getattr(self.client, name) + else: + raise AttributeError() + + def new_account_and_tos(self, regr, check_tos_cb=None): + """Combined register and agree_tos for V1, new_account for V2 + + :param .NewRegistration regr: + :param callable check_tos_cb: callback that raises an error if + the check does not work + """ + def _assess_tos(tos): + if check_tos_cb is not None: + check_tos_cb(tos) + if self.acme_version == 1: + regr = self.client.register(regr) + if regr.terms_of_service is not None: + _assess_tos(regr.terms_of_service) + return self.client.agree_to_tos(regr) + return regr + else: + if "terms_of_service" in self.client.directory.meta: + _assess_tos(self.client.directory.meta.terms_of_service) + regr = regr.update(terms_of_service_agreed=True) + return self.client.new_account(regr) + + def _acme_version_from_directory(self, directory): + if hasattr(directory, 'newNonce'): + return 2 + else: + return 1 + + class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Wrapper around requests that signs POSTs for authentication. diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 662c32942..eb03d9261 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -21,10 +21,25 @@ CERT_DER = test_util.load_vector('cert.der') KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) KEY2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) +DIRECTORY_V1 = messages.Directory({ + messages.NewRegistration: + 'https://www.letsencrypt-demo.org/acme/new-reg', + messages.Revocation: + 'https://www.letsencrypt-demo.org/acme/revoke-cert', + messages.NewAuthorization: + 'https://www.letsencrypt-demo.org/acme/new-authz', + messages.CertificateRequest: + 'https://www.letsencrypt-demo.org/acme/new-cert', +}) -class ClientTest(unittest.TestCase): - """Tests for acme.client.Client.""" - # pylint: disable=too-many-instance-attributes,too-many-public-methods +DIRECTORY_V2 = messages.Directory({ + 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', + 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce' +}) + + +class ClientTestBase(unittest.TestCase): + """Base for tests in acme.client.""" def setUp(self): self.response = mock.MagicMock( @@ -33,21 +48,6 @@ class ClientTest(unittest.TestCase): self.net.post.return_value = self.response self.net.get.return_value = self.response - self.directory = messages.Directory({ - messages.NewRegistration: - 'https://www.letsencrypt-demo.org/acme/new-reg', - messages.Revocation: - 'https://www.letsencrypt-demo.org/acme/revoke-cert', - messages.NewAuthorization: - 'https://www.letsencrypt-demo.org/acme/new-authz', - messages.CertificateRequest: - 'https://www.letsencrypt-demo.org/acme/new-cert', - }) - - from acme.client import Client - self.client = Client( - directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) - self.identifier = messages.Identifier( typ=messages.IDENTIFIER_FQDN, value='example.com') @@ -84,6 +84,124 @@ class ClientTest(unittest.TestCase): # Reason code for revocation self.rsn = 1 + +class BackwardsCompatibleClientV2Test(ClientTestBase): + """Tests for acme.client.BackwardsCompatibleClientV2.""" + + def _init(self): + uri = 'http://www.letsencrypt-demo.org/directory' + from acme.client import BackwardsCompatibleClientV2 + return BackwardsCompatibleClientV2(net=self.net, + key=KEY, server=uri) + + def test_init_downloads_directory(self): + uri = 'http://www.letsencrypt-demo.org/directory' + from acme.client import BackwardsCompatibleClientV2 + BackwardsCompatibleClientV2(net=self.net, + key=KEY, server=uri) + self.net.get.assert_called_once_with(uri) + + def test_init_acme_version(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + client = self._init() + self.assertEqual(client.acme_version, 1) + + self.response.json.return_value = DIRECTORY_V2.to_json() + client = self._init() + self.assertEqual(client.acme_version, 2) + + def test_forwarding(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + client = self._init() + self.assertEqual(client.directory, client.client.directory) + self.assertEqual(client.key, KEY) + # delete this line once we finish migrating to new API: + self.assertEqual(client.register, client.client.register) + self.assertEqual(client.update_registration, client.client.update_registration) + self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') + self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') + self.assertRaises(AttributeError, client.__getattr__, 'new_account') + + def test_new_account_and_tos(self): + # v2 no tos + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().new_account.assert_called_with(self.new_reg) + + # v2 tos good + with mock.patch('acme.client.ClientV2') as mock_client: + mock_client().directory.meta.__contains__.return_value = True + client = self._init() + client.new_account_and_tos(self.new_reg, lambda x: True) + mock_client().new_account.assert_called_with( + self.new_reg.update(terms_of_service_agreed=True)) + + # v2 tos bad + with mock.patch('acme.client.ClientV2') as mock_client: + mock_client().directory.meta.__contains__.return_value = True + client = self._init() + def _tos_cb(tos): + raise errors.Error + self.assertRaises(errors.Error, client.new_account_and_tos, + self.new_reg, _tos_cb) + mock_client().new_account.assert_not_called() + + # v1 yes tos + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + regr = mock.MagicMock(terms_of_service="TOS") + mock_client().register.return_value = regr + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().register.assert_called_once_with(self.new_reg) + mock_client().agree_to_tos.assert_called_once_with(regr) + + # v1 no tos + with mock.patch('acme.client.Client') as mock_client: + regr = mock.MagicMock(terms_of_service=None) + mock_client().register.return_value = regr + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().register.assert_called_once_with(self.new_reg) + mock_client().agree_to_tos.assert_not_called() + + +class ClientV2Test(ClientTestBase): + """Tests for acme.client.ClientV2.""" + # pylint: disable=too-many-instance-attributes,too-many-public-methods + + def setUp(self): + super(ClientV2Test, self).setUp() + from acme.client import ClientV2 + self.directory = DIRECTORY_V2 + self.client = ClientV2(directory=self.directory, net=self.net) + + def test_new_account_v2(self): + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.regr = messages.RegistrationResource( + body=messages.Registration( + contact=self.contact, key=KEY.public_key()), + uri='https://www.letsencrypt-demo.org/acme/reg/1') + + self.assertEqual(self.regr, self.client.new_account(self.regr)) + + +class ClientTest(ClientTestBase): + """Tests for acme.client.Client.""" + # pylint: disable=too-many-instance-attributes,too-many-public-methods + + def setUp(self): + super(ClientTest, self).setUp() + from acme.client import Client + self.directory = DIRECTORY_V1 + self.client = Client( + directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) + def test_init_downloads_directory(self): uri = 'http://www.letsencrypt-demo.org/directory' from acme.client import Client @@ -104,23 +222,6 @@ class ClientTest(unittest.TestCase): self.assertEqual(self.regr, self.client.register(self.new_reg)) # TODO: test POST call arguments - def test_new_account_v2(self): - directory = messages.Directory({ - "newAccount": 'https://www.letsencrypt-demo.org/acme/new-account', - }) - from acme.client import ClientV2 - client = ClientV2(directory, self.net) - self.response.status_code = http_client.CREATED - self.response.json.return_value = self.regr.body.to_json() - self.response.headers['Location'] = self.regr.uri - - self.regr = messages.RegistrationResource( - body=messages.Registration( - contact=self.contact, key=KEY.public_key()), - uri='https://www.letsencrypt-demo.org/acme/reg/1') - - self.assertEqual(self.regr, client.new_account(self.regr)) - def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: # pylint: disable=no-member diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 98993c4e1..21702f6a3 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -171,10 +171,31 @@ class Directory(jose.JSONDeSerializable): class Meta(jose.JSONObjectWithFields): """Directory Meta.""" - terms_of_service = jose.Field('terms-of-service', omitempty=True) + _terms_of_service = jose.Field('terms-of-service', omitempty=True) + _terms_of_service_v2 = jose.Field('termsOfService', omitempty=True) website = jose.Field('website', omitempty=True) caa_identities = jose.Field('caa-identities', omitempty=True) + def __init__(self, **kwargs): + kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items()) + # pylint: disable=star-args + super(Directory.Meta, self).__init__(**kwargs) + + @property + def terms_of_service(self): + """URL for the CA TOS""" + return self._terms_of_service or self._terms_of_service_v2 + + def __iter__(self): + # When iterating over fields, use the external name 'terms_of_service' instead of + # the internal '_terms_of_service'. + for name in super(Directory.Meta, self).__iter__(): + yield name[1:] if name == '_terms_of_service' else name + + def _internal_name(self, name): + return '_' + name if name == 'terms_of_service' else name + + @classmethod def _canon_key(cls, key): return getattr(key, 'resource_type', key) @@ -251,7 +272,7 @@ class Registration(ResourceBody): contact = jose.Field('contact', omitempty=True, default=()) agreement = jose.Field('agreement', omitempty=True) status = jose.Field('status', omitempty=True) - terms_of_service_agreed = jose.Field('terms-of-service-agreed', omitempty=True) + terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index fa885d3c2..4bc60a67b 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -165,6 +165,13 @@ class DirectoryTest(unittest.TestCase): from acme.messages import Directory Directory.from_json({'foo': 'bar'}) + def test_iter_meta(self): + result = False + for k in self.dir.meta: + if k == 'terms_of_service': + result = self.dir.meta[k] == 'https://example.com/acme/terms' + self.assertTrue(result) + class RegistrationTest(unittest.TestCase): """Tests for acme.messages.Registration.""" diff --git a/certbot/client.py b/certbot/client.py index bc25da549..67ee8f7fa 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -37,12 +37,12 @@ from certbot.plugins import selection as plugin_selection logger = logging.getLogger(__name__) -def acme_from_config_key(config, key): +def acme_from_config_key(config, key, regr=None): "Wrangle ACME client construction" # TODO: Allow for other alg types besides RS256 - net = acme_client.ClientNetwork(key, verify_ssl=(not config.no_verify_ssl), + net = acme_client.ClientNetwork(key, account=regr, verify_ssl=(not config.no_verify_ssl), user_agent=determine_user_agent(config)) - return acme_client.Client(config.server, key=key, net=net) + return acme_client.BackwardsCompatibleClientV2(net, key, config.server) def determine_user_agent(config): @@ -162,14 +162,7 @@ def register(config, account_storage, tos_cb=None): backend=default_backend()))) acme = acme_from_config_key(config, key) # TODO: add phone? - regr = perform_registration(acme, config) - - if regr.terms_of_service is not None: - if tos_cb is not None and not tos_cb(regr): - raise errors.Error( - "Registration cannot proceed without accepting " - "Terms of Service.") - regr = acme.agree_to_tos(regr) + regr = perform_registration(acme, config, tos_cb) acc = account.Account(regr, key) account.report_new_account(config) @@ -180,7 +173,7 @@ def register(config, account_storage, tos_cb=None): return acc, acme -def perform_registration(acme, config): +def perform_registration(acme, config, tos_cb): """ Actually register new account, trying repeatedly if there are email problems @@ -192,7 +185,8 @@ def perform_registration(acme, config): :rtype: `acme.messages.RegistrationResource` """ try: - return acme.register(messages.NewRegistration.from_data(email=config.email)) + return acme.new_account_and_tos(messages.NewRegistration.from_data(email=config.email), + tos_cb) except messages.Error as e: if e.code == "invalidEmail" or e.code == "invalidContact": if config.noninteractive_mode: @@ -202,7 +196,7 @@ def perform_registration(acme, config): raise errors.Error(msg) else: config.email = display_ops.get_email(invalid=True) - return perform_registration(acme, config) + return perform_registration(acme, config, tos_cb) else: raise @@ -232,7 +226,7 @@ class Client(object): # Initialize ACME if account is provided if acme is None and self.account is not None: - acme = acme_from_config_key(config, self.account.key) + acme = acme_from_config_key(config, self.account.key, self.account.regr) self.acme = acme if auth is not None: diff --git a/certbot/main.py b/certbot/main.py index 13234e0d2..ff3758985 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -496,17 +496,20 @@ def _determine_account(config): if config.email is None and not config.register_unsafely_without_email: config.email = display_ops.get_email() - def _tos_cb(regr): + def _tos_cb(terms_of_service): if config.tos: return True msg = ("Please read the Terms of Service at {0}. You " "must agree in order to register with the ACME " "server at {1}".format( - regr.terms_of_service, config.server)) + terms_of_service, config.server)) obj = zope.component.getUtility(interfaces.IDisplay) - return obj.yesno(msg, "Agree", "Cancel", + result = obj.yesno(msg, "Agree", "Cancel", cli_flag="--agree-tos", force_interactive=True) - + if not result: + raise errors.Error( + "Registration cannot proceed without accepting " + "Terms of Service.") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 204f46323..a9a87b80b 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -30,31 +30,27 @@ class RegisterTest(test_util.ConfigTestCase): self.config.register_unsafely_without_email = False self.config.email = "alias@example.com" self.account_storage = account.AccountMemoryStorage() - self.tos_cb = mock.MagicMock() def _call(self): from certbot.client import register - return register(self.config, self.account_storage, self.tos_cb) + tos_cb = mock.MagicMock() + return register(self.config, self.account_storage, tos_cb) def test_no_tos(self): - with mock.patch("certbot.client.acme_client.Client") as mock_client: - mock_client.register().terms_of_service = "http://tos" + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client.new_account_and_tos().terms_of_service = "http://tos" with mock.patch("certbot.eff.handle_subscription") as mock_handle: with mock.patch("certbot.account.report_new_account"): - self.tos_cb.return_value = False + mock_client().new_account_and_tos.side_effect = errors.Error self.assertRaises(errors.Error, self._call) self.assertFalse(mock_handle.called) - self.tos_cb.return_value = True + mock_client().new_account_and_tos.side_effect = None self._call() self.assertTrue(mock_handle.called) - self.tos_cb = None - self._call() - self.assertEqual(mock_handle.call_count, 2) - def test_it(self): - with mock.patch("certbot.client.acme_client.Client"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): with mock.patch("certbot.account.report_new_account"): with mock.patch("certbot.eff.handle_subscription"): self._call() @@ -66,9 +62,9 @@ class RegisterTest(test_util.ConfigTestCase): self.config.noninteractive_mode = False msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription") as mock_handle: - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self._call() self.assertEqual(mock_get_email.call_count, 1) self.assertTrue(mock_handle.called) @@ -79,9 +75,9 @@ class RegisterTest(test_util.ConfigTestCase): self.config.noninteractive_mode = True msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription"): - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(errors.Error, self._call) def test_needs_email(self): @@ -91,7 +87,7 @@ class RegisterTest(test_util.ConfigTestCase): @mock.patch("certbot.client.logger") def test_without_email(self, mock_logger): with mock.patch("certbot.eff.handle_subscription") as mock_handle: - with mock.patch("certbot.client.acme_client.Client"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): with mock.patch("certbot.account.report_new_account"): self.config.email = None self.config.register_unsafely_without_email = True @@ -104,9 +100,9 @@ class RegisterTest(test_util.ConfigTestCase): from acme import messages msg = "Test" mx_err = messages.Error(detail=msg, typ="malformed", title="title") - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription") as mock_handle: - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(messages.Error, self._call) self.assertFalse(mock_handle.called) @@ -122,7 +118,7 @@ class ClientTestCommon(test_util.ConfigTestCase): self.account = mock.MagicMock(**{"key.pem": KEY}) from certbot.client import Client - with mock.patch("certbot.client.acme_client.Client") as acme: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as acme: self.acme_client = acme self.acme = acme.return_value = mock.MagicMock() self.client = Client( @@ -140,7 +136,7 @@ class ClientTest(ClientTestCommon): self.eg_domains = ["example.com", "www.example.com"] def test_init_acme_verify_ssl(self): - net = self.acme_client.call_args[1]["net"] + net = self.acme_client.call_args[0][0] self.assertTrue(net.verify_ssl) def _mock_obtain_certificate(self): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index e84de54d0..518653a53 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -225,7 +225,7 @@ class RevokeTest(test_util.TempDirTestCase): 'cert_512.pem')) self.patches = [ - mock.patch('acme.client.Client', autospec=True), + mock.patch('acme.client.BackwardsCompatibleClientV2'), mock.patch('certbot.client.Client'), mock.patch('certbot.main._determine_account'), mock.patch('certbot.main.display_ops.success_revocation') @@ -267,7 +267,7 @@ class RevokeTest(test_util.TempDirTestCase): def test_revoke_with_reason(self, mock_acme_client, mock_delete_if_appropriate): mock_delete_if_appropriate.return_value = False - mock_revoke = mock_acme_client.Client().revoke + mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke expected = [] for reason, code in constants.REVOCATION_REASONS.items(): self._call("--reason " + reason) @@ -661,10 +661,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._cli_missing_flag(args, "specify a plugin") args.extend(['--standalone', '-d', 'eg.is']) self._cli_missing_flag(args, "register before running") - with mock.patch('certbot.main._get_and_save_cert'): - with mock.patch('certbot.main.client.acme_from_config_key'): - args.extend(['--email', 'io@io.is']) - self._cli_missing_flag(args, "--agree-tos") @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.main.client.acme_client.Client') @@ -693,7 +689,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met ua = "bandersnatch" args += ["--user-agent", ua] self._call_no_clientmock(args) - acme_net.assert_called_once_with(mock.ANY, verify_ssl=True, user_agent=ua) + acme_net.assert_called_once_with(mock.ANY, account=mock.ANY, verify_ssl=True, + user_agent=ua) @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') @@ -1352,11 +1349,11 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call_no_clientmock(['--cert-path', SS_CERT_PATH, '--key-path', RSA2048_KEY_PATH, '--server', server, 'revoke']) with open(RSA2048_KEY_PATH, 'rb') as f: - mock_acme_client.Client.assert_called_once_with( - server, key=jose.JWK.load(f.read()), net=mock.ANY) + mock_acme_client.BackwardsCompatibleClientV2.assert_called_once_with( + mock.ANY, jose.JWK.load(f.read()), server) with open(SS_CERT_PATH, 'rb') as f: cert = crypto_util.pyopenssl_load_certificate(f.read())[0] - mock_revoke = mock_acme_client.Client().revoke + mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke mock_revoke.assert_called_once_with( jose.ComparableX509(cert), mock.ANY) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index ddd4a1aec..60d8d6084 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -340,7 +340,7 @@ class ConfigTestCase(TempDirTestCase): self.config.cert_path = constants.CLI_DEFAULTS['auth_cert_path'] self.config.fullchain_path = constants.CLI_DEFAULTS['auth_chain_path'] self.config.chain_path = constants.CLI_DEFAULTS['auth_chain_path'] - self.config.server = "example.com" + self.config.server = "https://example.com" def lock_and_call(func, lock_path): """Grab a lock for lock_path and call func. From e48898a8c82e995470b909f13cc020ae7c32b5cd Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 14 Feb 2018 19:19:23 -0800 Subject: [PATCH 56/98] ACMEv2: Add Order support This adds two new classes in messages: Order and OrderResource. It also adds methods to ClientV2 to create orders, and poll orders then request issuance. The CSR is stored on the OrderResource so it can be carried along and submitted when it's time to finalize the order. --- acme/acme/client.py | 98 ++++++++++++++++++++++++++++++++++---- acme/acme/errors.py | 21 ++++++++ acme/acme/messages.py | 48 ++++++++++++++++++- acme/acme/messages_test.py | 18 ++++++- 4 files changed, 173 insertions(+), 12 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index d843feaa7..219667d53 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,6 +1,7 @@ """ACME client API.""" import base64 import collections +import cryptography import datetime from email.utils import parsedate_tz import heapq @@ -119,11 +120,11 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes """ return self._send_recv_regr(regr, messages.UpdateRegistration()) - def _authzr_from_response(self, response, identifier, uri=None): + def _authzr_from_response(self, response, identifier=None, uri=None): authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), uri=response.headers.get('Location', uri)) - if authzr.body.identifier != identifier: + if identifier is not None and authzr.body.identifier != identifier: raise errors.UnexpectedUpdate(authzr) return authzr @@ -233,8 +234,8 @@ class Client(ClientBase): instances of `.DeserializationError` raised in `from_json()`. :ivar messages.Directory directory: - :ivar key: `.JWK` (private) - :ivar alg: `.JWASignature` + :ivar key: `josepy.JWK` (private) + :ivar alg: `josepy.JWASignature` :ivar bool verify_ssl: Verify SSL certificates? :ivar .ClientNetwork net: Client network. Useful for testing. If not supplied, it will be initialized using `key`, `alg` and @@ -550,7 +551,6 @@ class ClientV2(ClientBase): :returns: Registration Resource. :rtype: `.RegistrationResource` - """ response = self.net.post(self.directory['newAccount'], new_account, acme_version=2) @@ -560,6 +560,84 @@ class ClientV2(ClientBase): self.net.account = regr return regr + def new_order(self, csr_pem): + """Request a new Order object from the server. + + :param str csr_pem: A CSR in PEM format. + + :returns: The newly created order. + :rtype: OrderResource + """ + csr = cryptography.x509.load_pem_x509_csr(csr_pem, + cryptography.hazmat.backends.default_backend()) + san_extension = next(ext for ext in csr.extensions + if ext.oid == cryptography.x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) + dnsNames = san_extension.value.get_values_for_type(cryptography.x509.DNSName) + + identifiers = [] + for name in dnsNames: + identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN, + value=name)) + order = messages.NewOrder(identifiers=identifiers) + response = self.net.post(self.directory['newOrder'], order) + body = messages.Order.from_json(response.json()) + authorizations = [] + for url in body.authorizations: + authorizations.append(self._authzr_from_response(self.net.get(url))) + return messages.OrderResource( + body=body, + uri=response.headers.get('Location', uri), + fullchain_pem=fullchain_pem, + authorizations=authorizations, + csr_pem=csr_pem) + + def poll_and_finalize(self, orderr, deadline=None): + if deadline is None: + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.poll_authorizations(orderr, deadline) + return self.finalize_order(orderr, deadline) + + def poll_authorizations(self, orderr, deadline): + """Poll Order Resource for status.""" + responses = [] + for url in orderr.body.authorizations: + while datetime.datetime.now() < deadline: + authzr = self._authzr_from_response(self.net.get(url), uri=url) + if authzr.body.status != messages.STATUS_PENDING: + responses.append(authzr) + break + time.sleep(1) + # If we didn't get a response for every authorization, we fell through + # the bottom of the loop due to hitting the deadline. + if len(responses) > orderr.body.authorizations: + raise TimeoutError() + failed = [] + for authzr in responses: + if authzr.body.status != messages.STATUS_VALID: + for chall in authzr.body.challenges: + if chall.error != None: + failed.append(authzr) + if len(failed) > 0: + raise ValidationError(failed) + return orderr.update(authorizations=responses) + + def finalize_order(self, orderr, deadline): + csr = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) + wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) + self.net.post(latest.body.finalize, wrapped_csr) + while datetime.datetime.now() < deadline: + time.sleep(1) + response = self.net.get(orderr.uri) + body = messages.Order.from_json(response.json()) + if body.error is not None: + raise IssuanceError(body.error) + if body.certificate is not None: + certificate_response = self.net.get(body.certificate).text + return orderr.update(fullchain_pem=certificate_response) + raise TimeoutError() + + class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but supports V1 servers. @@ -628,10 +706,10 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Initialize. - :param key: Account private key + :param josepy.JWK key: Account private key :param messages.RegistrationResource account: Account object. Required if you are - planning to use .post() with acme_version=2 for anything other than creating a new - account; may be set later after registering. + planning to use .post() with acme_version=2 for anything other than + creating a new account; may be set later after registering. :param josepy.JWASignature alg: Algoritm to use in signing JWS. :param bool verify_ssl: Whether to verify certificates on SSL connections. :param str user_agent: String to send as User-Agent header. @@ -662,10 +740,10 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes .. todo:: Implement ``acmePath``. - :param .JSONDeSerializable obj: + :param josepy.JSONDeSerializable obj: :param str url: The URL to which this object will be POSTed :param bytes nonce: - :rtype: `.JWS` + :rtype: `josepy.JWS` """ jobj = obj.json_dumps(indent=2).encode() diff --git a/acme/acme/errors.py b/acme/acme/errors.py index de5f9d1f4..624ccb3d9 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -83,6 +83,27 @@ class PollError(ClientError): return '{0}(exhausted={1!r}, updated={2!r})'.format( self.__class__.__name__, self.exhausted, self.updated) +class ValidationError(Error): + """Error for authorization failures. Contains a list of authorization + resources, each of which is invalid and should have an error field. + """ + def __init__(self, failed_authzrs): + self.failed_authzrs = failed_authzrs + super(ClientError, self).__init__() + +class TimeoutError(Error): + """Error for when polling an authorization or an order times out.""" + +class IssuanceError(Error): + """Error sent by the server after requesting issuance of a certificate.""" + + def __init__(self, error): + """Initialize. + + :param messages.Error error: The error provided by the server. + """ + self.error = error + class ConflictError(ClientError): """Error for when the server returns a 409 (Conflict) HTTP status. diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 21702f6a3..418bcead9 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -174,7 +174,7 @@ class Directory(jose.JSONDeSerializable): _terms_of_service = jose.Field('terms-of-service', omitempty=True) _terms_of_service_v2 = jose.Field('termsOfService', omitempty=True) website = jose.Field('website', omitempty=True) - caa_identities = jose.Field('caa-identities', omitempty=True) + caa_identities = jose.Field('caaIdentities', omitempty=True) def __init__(self, **kwargs): kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items()) @@ -504,3 +504,49 @@ class Revocation(jose.JSONObjectWithFields): certificate = jose.Field( 'certificate', decoder=jose.decode_cert, encoder=jose.encode_cert) reason = jose.Field('reason') + + +class Order(ResourceBody): + """Order Resource Body. + + .. note:: Parsing of identifiers on response doesn't work right now; to make + it work we would need to set up the equivalent of Identifier.from_json, but + for a list. + :ivar list of .Identifier: List of identifiers for the certificate. + :ivar acme.messages.Status status: + :ivar list of str authorizations: URLs of authorizations. + :ivar str certificate: URL to download certificate as a fullchain PEM. + :ivar str finalize: URL to POST to to request issuance once all + authorizations have "valid" status. + :ivar datetime.datetime expires: When the order expires. + :ivar .Error error: Any error that occurred during finalization, if applicable. + """ + identifiers = jose.Field('identifiers', omitempty=True) + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) + authorizations = jose.Field('authorizations', omitempty=True) + certificate = jose.Field('certificate', omitempty=True) + finalize = jose.Field('finalize', omitempty=True) + expires = fields.RFC3339Field('expires', omitempty=True) + error = jose.Field('error', omitempty=True, decoder=Error.from_json) + +class OrderResource(ResourceWithURI): + """Order Resource. + + :ivar acme.messages.Order body: + :ivar str csr_pem: The CSR this Order will be finalized with. + :ivar list of acme.messages.AuthorizationResource authorizations: + Fully-fetched AuthorizationResource objects. + :ivar str fullchain_pem: The fetched contents of the certificate URL + produced once the order was finalized, if it's present. + """ + body = jose.Field('body', decoder=Order.from_json) + csr_pem = jose.Field('csr_pem', omitempty=True) + authorizations = jose.Field('authorizations') + fullchain_pem = jose.Field('fullchain_pem', omitempty=True) + +@Directory.register +class NewOrder(Order): + """New order.""" + resource_type = 'new-order' + resource = fields.Resource(resource_type) diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 4bc60a67b..64bc81efd 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -157,7 +157,7 @@ class DirectoryTest(unittest.TestCase): 'meta': { 'terms-of-service': 'https://example.com/acme/terms', 'website': 'https://www.example.com/', - 'caa-identities': ['example.com'], + 'caaIdentities': ['example.com'], }, }) @@ -408,5 +408,21 @@ class RevocationTest(unittest.TestCase): hash(Revocation.from_json(self.rev.to_json())) +class OrderResourceTest(unittest.TestCase): + """Tests for acme.messages.OrderResource.""" + + def setUp(self): + from acme.messages import OrderResource + self.regr = OrderResource( + body=mock.sentinel.body, uri=mock.sentinel.uri) + + def test_to_partial_json(self): + self.assertEqual(self.regr.to_json(), { + 'body': mock.sentinel.body, + 'uri': mock.sentinel.uri, + 'authorizations': None, + }) + + if __name__ == '__main__': unittest.main() # pragma: no cover From 70a75ebe9dffb7d59e701e2dcb04ce23a1bcc0fc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Feb 2018 19:40:14 -0800 Subject: [PATCH 57/98] Add tests and fix minor bugs in Order support * delint * refactor client tests * Add test for new order and fix identifiers parsing. * Add poll_and_finalize test * Test and fix poll_authorizations timeout * Add test_failed_authorizations * Add test_poll_authorizations_success * Test and fix finalize_order success * add test_finalize_order_timeout * add test_finalize_order_error * test sleep code --- acme/acme/client.py | 38 ++++++-- acme/acme/client_test.py | 181 +++++++++++++++++++++++++++++++-------- acme/acme/errors.py | 3 +- acme/acme/messages.py | 7 +- 4 files changed, 180 insertions(+), 49 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 219667d53..bc93ca06f 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -586,12 +586,23 @@ class ClientV2(ClientBase): authorizations.append(self._authzr_from_response(self.net.get(url))) return messages.OrderResource( body=body, - uri=response.headers.get('Location', uri), - fullchain_pem=fullchain_pem, + uri=response.headers.get('Location'), authorizations=authorizations, csr_pem=csr_pem) def poll_and_finalize(self, orderr, deadline=None): + """Poll authorizations and finalize the order. + + If no deadline is provided, this method will timeout after 90 + seconds. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ if deadline is None: deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) orderr = self.poll_authorizations(orderr, deadline) @@ -609,8 +620,8 @@ class ClientV2(ClientBase): time.sleep(1) # If we didn't get a response for every authorization, we fell through # the bottom of the loop due to hitting the deadline. - if len(responses) > orderr.body.authorizations: - raise TimeoutError() + if len(responses) < len(orderr.body.authorizations): + raise errors.TimeoutError() failed = [] for authzr in responses: if authzr.body.status != messages.STATUS_VALID: @@ -618,24 +629,33 @@ class ClientV2(ClientBase): if chall.error != None: failed.append(authzr) if len(failed) > 0: - raise ValidationError(failed) + raise errors.ValidationError(failed) return orderr.update(authorizations=responses) def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) - self.net.post(latest.body.finalize, wrapped_csr) + self.net.post(orderr.body.finalize, wrapped_csr) while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) body = messages.Order.from_json(response.json()) if body.error is not None: - raise IssuanceError(body.error) + raise errors.IssuanceError(body.error) if body.certificate is not None: certificate_response = self.net.get(body.certificate).text - return orderr.update(fullchain_pem=certificate_response) - raise TimeoutError() + return orderr.update(body=body, fullchain_pem=certificate_response) + raise errors.TimeoutError() class BackwardsCompatibleClientV2(object): diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index eb03d9261..11516c02f 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -1,4 +1,5 @@ """Tests for acme.client.""" +import copy import datetime import json import unittest @@ -18,6 +19,8 @@ from acme import test_util CERT_DER = test_util.load_vector('cert.der') +CERT_SAN_PEM = test_util.load_vector('cert-san.pem') +CSR_SAN_PEM = test_util.load_vector('csr-san.pem') KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) KEY2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) @@ -34,7 +37,8 @@ DIRECTORY_V1 = messages.Directory({ DIRECTORY_V2 = messages.Directory({ 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', - 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce' + 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce', + 'newOrder': 'https://www.letsencrypt-demo.org/acme/new-order', }) @@ -57,8 +61,7 @@ class ClientTestBase(unittest.TestCase): contact=self.contact, key=KEY.public_key()) self.new_reg = messages.NewRegistration(**dict(reg)) self.regr = messages.RegistrationResource( - body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1', - terms_of_service='https://www.letsencrypt-demo.org/tos') + body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') # Authorization authzr_uri = 'https://www.letsencrypt-demo.org/acme/authz/1' @@ -75,15 +78,6 @@ class ClientTestBase(unittest.TestCase): self.authzr = messages.AuthorizationResource( body=self.authz, uri=authzr_uri) - # Request issuance - self.certr = messages.CertificateResource( - body=messages_test.CERT, authzrs=(self.authzr,), - uri='https://www.letsencrypt-demo.org/acme/cert/1', - cert_chain_uri='https://www.letsencrypt-demo.org/ca') - - # Reason code for revocation - self.rsn = 1 - class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" @@ -168,37 +162,29 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().agree_to_tos.assert_not_called() -class ClientV2Test(ClientTestBase): - """Tests for acme.client.ClientV2.""" - # pylint: disable=too-many-instance-attributes,too-many-public-methods - - def setUp(self): - super(ClientV2Test, self).setUp() - from acme.client import ClientV2 - self.directory = DIRECTORY_V2 - self.client = ClientV2(directory=self.directory, net=self.net) - - def test_new_account_v2(self): - self.response.status_code = http_client.CREATED - self.response.json.return_value = self.regr.body.to_json() - self.response.headers['Location'] = self.regr.uri - - self.regr = messages.RegistrationResource( - body=messages.Registration( - contact=self.contact, key=KEY.public_key()), - uri='https://www.letsencrypt-demo.org/acme/reg/1') - - self.assertEqual(self.regr, self.client.new_account(self.regr)) - - class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" # pylint: disable=too-many-instance-attributes,too-many-public-methods def setUp(self): super(ClientTest, self).setUp() - from acme.client import Client + self.directory = DIRECTORY_V1 + + # Registration + self.regr = self.regr.update( + terms_of_service='https://www.letsencrypt-demo.org/tos') + + # Request issuance + self.certr = messages.CertificateResource( + body=messages_test.CERT, authzrs=(self.authzr,), + uri='https://www.letsencrypt-demo.org/acme/cert/1', + cert_chain_uri='https://www.letsencrypt-demo.org/ca') + + # Reason code for revocation + self.rsn = 1 + + from acme.client import Client self.client = Client( directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) @@ -554,6 +540,129 @@ class ClientTest(ClientTestBase): self.certr, self.rsn) +class ClientV2Test(ClientTestBase): + """Tests for acme.client.ClientV2.""" + + def setUp(self): + super(ClientV2Test, self).setUp() + + self.directory = DIRECTORY_V2 + + from acme.client import ClientV2 + self.client = ClientV2(self.directory, self.net) + + self.new_reg = self.new_reg.update(terms_of_service_agreed=True) + + self.authzr_uri2 = 'https://www.letsencrypt-demo.org/acme/authz/2' + self.authz2 = self.authz.update(identifier=messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value='www.example.com'), + status=messages.STATUS_PENDING) + self.authzr2 = messages.AuthorizationResource( + body=self.authz2, uri=self.authzr_uri2) + + self.order = messages.Order( + identifiers=(self.authz.identifier, self.authz2.identifier), + status=messages.STATUS_PENDING, + authorizations=(self.authzr.uri, self.authzr_uri2), + finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') + self.orderr = messages.OrderResource( + body=self.order, + uri='https://www.letsencrypt-demo.org/acme/acct/1/order/1', + authorizations=[self.authzr, self.authzr2], csr_pem=CSR_SAN_PEM) + + def test_new_account(self): + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.assertEqual(self.regr, self.client.new_account(self.new_reg)) + + def test_new_order(self): + order_response = copy.deepcopy(self.response) + order_response.status_code = http_client.CREATED + order_response.json.return_value = self.order.to_json() + order_response.headers['Location'] = self.orderr.uri + self.net.post.return_value = order_response + + authz_response = copy.deepcopy(self.response) + authz_response.json.return_value = self.authz.to_json() + authz_response.headers['Location'] = self.authzr.uri + authz_response2 = self.response + authz_response2.json.return_value = self.authz2.to_json() + authz_response2.headers['Location'] = self.authzr2.uri + self.net.get.side_effect = (authz_response, authz_response2) + + self.assertEqual(self.client.new_order(CSR_SAN_PEM), self.orderr) + + @mock.patch('acme.client.datetime') + def test_poll_and_finalize(self, mock_datetime): + mock_datetime.datetime.now.return_value = datetime.datetime(2018, 2, 15) + mock_datetime.timedelta = datetime.timedelta + expected_deadline = mock_datetime.datetime.now() + datetime.timedelta(seconds=90) + + self.client.poll_authorizations = mock.Mock(return_value=self.orderr) + self.client.finalize_order = mock.Mock(return_value=self.orderr) + + self.assertEqual(self.client.poll_and_finalize(self.orderr), self.orderr) + self.client.poll_authorizations.assert_called_once_with(self.orderr, expected_deadline) + self.client.finalize_order.assert_called_once_with(self.orderr, expected_deadline) + + @mock.patch('acme.client.datetime') + def test_poll_authorizations_timeout(self, mock_datetime): + now_side_effect = [datetime.datetime(2018, 2, 15), + datetime.datetime(2018, 2, 16), + datetime.datetime(2018, 2, 17)] + mock_datetime.datetime.now.side_effect = now_side_effect + self.response.json.side_effect = [ + self.authz.to_json(), self.authz2.to_json(), self.authz2.to_json()] + + self.assertRaises( + errors.TimeoutError, self.client.poll_authorizations, self.orderr, now_side_effect[1]) + + def test_poll_authorizations_failure(self): + deadline = datetime.datetime(9999, 9, 9) + challb = self.challr.body.update(status=messages.STATUS_INVALID, + error=messages.Error.with_code('unauthorized')) + authz = self.authz.update(status=messages.STATUS_INVALID, challenges=(challb,)) + self.response.json.return_value = authz.to_json() + + self.assertRaises( + errors.ValidationError, self.client.poll_authorizations, self.orderr, deadline) + + def test_poll_authorizations_success(self): + deadline = datetime.datetime(9999, 9, 9) + updated_authz2 = self.authz2.update(status=messages.STATUS_VALID) + updated_authzr2 = messages.AuthorizationResource( + body=updated_authz2, uri=self.authzr_uri2) + updated_orderr = self.orderr.update(authorizations=[self.authzr, updated_authzr2]) + + self.response.json.side_effect = ( + self.authz.to_json(), self.authz2.to_json(), updated_authz2.to_json()) + self.assertEqual(self.client.poll_authorizations(self.orderr, deadline), updated_orderr) + + def test_finalize_order_success(self): + updated_order = self.order.update( + certificate='https://www.letsencrypt-demo.org/acme/cert/') + updated_orderr = self.orderr.update(body=updated_order, fullchain_pem=CERT_SAN_PEM) + + self.response.json.return_value = updated_order.to_json() + self.response.text = CERT_SAN_PEM + + deadline = datetime.datetime(9999, 9, 9) + self.assertEqual(self.client.finalize_order(self.orderr, deadline), updated_orderr) + + def test_finalize_order_error(self): + updated_order = self.order.update(error=messages.Error.with_code('unauthorized')) + self.response.json.return_value = updated_order.to_json() + + deadline = datetime.datetime(9999, 9, 9) + self.assertRaises(errors.IssuanceError, self.client.finalize_order, self.orderr, deadline) + + def test_finalize_order_timeout(self): + deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + self.assertRaises(errors.TimeoutError, self.client.finalize_order, self.orderr, deadline) + + class MockJSONDeSerializable(jose.JSONDeSerializable): # pylint: disable=missing-docstring def __init__(self, value): diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 624ccb3d9..991335958 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -89,7 +89,7 @@ class ValidationError(Error): """ def __init__(self, failed_authzrs): self.failed_authzrs = failed_authzrs - super(ClientError, self).__init__() + super(ValidationError, self).__init__() class TimeoutError(Error): """Error for when polling an authorization or an order times out.""" @@ -103,6 +103,7 @@ class IssuanceError(Error): :param messages.Error error: The error provided by the server. """ self.error = error + super(IssuanceError, self).__init__() class ConflictError(ClientError): """Error for when the server returns a 409 (Conflict) HTTP status. diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 418bcead9..23cd66c63 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -509,9 +509,6 @@ class Revocation(jose.JSONObjectWithFields): class Order(ResourceBody): """Order Resource Body. - .. note:: Parsing of identifiers on response doesn't work right now; to make - it work we would need to set up the equivalent of Identifier.from_json, but - for a list. :ivar list of .Identifier: List of identifiers for the certificate. :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. @@ -530,6 +527,10 @@ class Order(ResourceBody): expires = fields.RFC3339Field('expires', omitempty=True) error = jose.Field('error', omitempty=True, decoder=Error.from_json) + @identifiers.decoder + def identifiers(value): # pylint: disable=missing-docstring,no-self-argument + return tuple(Identifier.from_json(identifier) for identifier in value) + class OrderResource(ResourceWithURI): """Order Resource. From adec7a8feda1b6ad4f989d4c293fdb78a646917a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Feb 2018 09:51:27 -0800 Subject: [PATCH 58/98] Cleanup dockerfile-dev (#5435) * cleanup dockerfile-dev * map port 80 * remove python3-dev package --- Dockerfile-dev | 65 ++++++---------------------------------------- docker-compose.yml | 1 + 2 files changed, 9 insertions(+), 57 deletions(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 581b58f11..9e35ebec8 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,70 +1,21 @@ # This Dockerfile builds an image for development. -FROM ubuntu:trusty -MAINTAINER Jakub Warmuz -MAINTAINER William Budington -MAINTAINER Yan +FROM ubuntu:xenial -# Note: this only exposes the port to other docker containers. You -# still have to bind to 443@host at runtime, as per the ACME spec. -EXPOSE 443 - -# TODO: make sure --config-dir and --work-dir cannot be changed -# through the CLI (certbot-docker wrapper that uses standalone -# authenticator and text mode only?) -VOLUME /etc/letsencrypt /var/lib/letsencrypt +# Note: this only exposes the port to other docker containers. +EXPOSE 80 443 WORKDIR /opt/certbot/src -# no need to mkdir anything: -# https://docs.docker.com/reference/builder/#copy -# If doesn't exist, it is created along with all missing -# directories in its path. - # TODO: Install Apache/Nginx for plugin development. -COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto -RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ - apt-get install python3-dev git -y && \ +COPY . . +RUN apt-get update && \ + apt-get install apache2 git nginx-light -y && \ + letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ /var/tmp/* -# the above is not likely to change, so by putting it further up the -# Dockerfile we make sure we cache as much as possible - -COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ - -# all above files are necessary for setup.py, however, package source -# code directory has to be copied separately to a subdirectory... -# https://docs.docker.com/reference/builder/#copy: "If is a -# directory, the entire contents of the directory are copied, -# including filesystem metadata. Note: The directory itself is not -# copied, just its contents." Order again matters, three files are far -# more likely to be cached than the whole project directory - -COPY certbot /opt/certbot/src/certbot/ -COPY acme /opt/certbot/src/acme/ -COPY certbot-apache /opt/certbot/src/certbot-apache/ -COPY certbot-nginx /opt/certbot/src/certbot-nginx/ -COPY letshelp-certbot /opt/certbot/src/letshelp-certbot/ -COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ -COPY tests /opt/certbot/src/tests/ - -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ - /opt/certbot/venv/bin/pip install -U pip && \ - /opt/certbot/venv/bin/pip install -U setuptools && \ - /opt/certbot/venv/bin/pip install \ - -e /opt/certbot/src/acme \ - -e /opt/certbot/src \ - -e /opt/certbot/src/certbot-apache \ - -e /opt/certbot/src/certbot-nginx \ - -e /opt/certbot/src/letshelp-certbot \ - -e /opt/certbot/src/certbot-compatibility-test \ - -e /opt/certbot/src[dev,docs] - -# install in editable mode (-e) to save space: it's not possible to -# "rm -rf /opt/certbot/src" (it's stays in the underlaying image); -# this might also help in debugging: you can "docker run --entrypoint -# bash" and investigate, apply patches, etc. +RUN VENV_NAME="../venv" tools/venv.sh ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/docker-compose.yml b/docker-compose.yml index 00d3d4c72..75a5b9aab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ services: context: . dockerfile: Dockerfile-dev ports: + - "80:80" - "443:443" volumes: - .:/opt/certbot/src From 2a142aa93288d0db87b8f12dc71fce70ee3ce482 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Feb 2018 14:47:10 -0800 Subject: [PATCH 59/98] Make Certbot depend on josepy (#5542) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ce505a62e..47b5b0b2c 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ install_requires = [ 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=1.2', # load_pem_x509_certificate + 'josepy', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', From e95e963ad62735a0cd23a46e27a7cbef3790afb2 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 16 Feb 2018 16:05:16 -0800 Subject: [PATCH 60/98] Get common name from CSR in new_order in ClientV2 (#5587) * switch new_order to use crypto_util._pyopenssl_cert_or_req_san * move certbot.crypto_util._get_names_from_loaded_cert_or_req functionality to acme.crypto_util._pyopenssl_cert_or_req_all_names --- acme/acme/client.py | 10 ++++------ acme/acme/crypto_util.py | 9 +++++++++ acme/acme/crypto_util_test.py | 24 ++++++++++++++++++++++++ acme/acme/testdata/cert-nocn.der | Bin 0 -> 1397 bytes certbot/crypto_util.py | 8 +------- 5 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 acme/acme/testdata/cert-nocn.der diff --git a/acme/acme/client.py b/acme/acme/client.py index bc93ca06f..1f4ae4fad 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,7 +1,6 @@ """ACME client API.""" import base64 import collections -import cryptography import datetime from email.utils import parsedate_tz import heapq @@ -17,6 +16,7 @@ import re import requests import sys +from acme import crypto_util from acme import errors from acme import jws from acme import messages @@ -568,11 +568,9 @@ class ClientV2(ClientBase): :returns: The newly created order. :rtype: OrderResource """ - csr = cryptography.x509.load_pem_x509_csr(csr_pem, - cryptography.hazmat.backends.default_backend()) - san_extension = next(ext for ext in csr.extensions - if ext.oid == cryptography.x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) - dnsNames = san_extension.value.get_values_for_type(cryptography.x509.DNSName) + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) identifiers = [] for name in dnsNames: diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index b8fba0348..a986721f0 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -186,6 +186,15 @@ def make_csr(private_key_pem, domains, must_staple=False): return OpenSSL.crypto.dump_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr) +def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): + common_name = loaded_cert_or_req.get_subject().CN + sans = _pyopenssl_cert_or_req_san(loaded_cert_or_req) + + if common_name is None: + return sans + else: + return [common_name] + [d for d in sans if d != common_name] + def _pyopenssl_cert_or_req_san(cert_or_req): """Get Subject Alternative Names from certificate or CSR using pyOpenSSL. diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf..14aaac8b5 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -65,6 +65,30 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # self.assertRaises(errors.Error, self._probe, b'bar') +class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): + """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" + + @classmethod + def _call(cls, loader, name): + # pylint: disable=protected-access + from acme.crypto_util import _pyopenssl_cert_or_req_all_names + return _pyopenssl_cert_or_req_all_names(loader(name)) + + def _call_cert(self, name): + return self._call(test_util.load_cert, name) + + def test_cert_one_san_no_common(self): + self.assertEqual(self._call_cert('cert-nocn.der'), + ['no-common-name.badssl.com']) + + def test_cert_no_sans_yes_common(self): + self.assertEqual(self._call_cert('cert.pem'), ['example.com']) + + def test_cert_two_sans_yes_common(self): + self.assertEqual(self._call_cert('cert-san.pem'), + ['example.com', 'www.example.com']) + + class PyOpenSSLCertOrReqSANTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_san.""" diff --git a/acme/acme/testdata/cert-nocn.der b/acme/acme/testdata/cert-nocn.der new file mode 100644 index 0000000000000000000000000000000000000000..59da83ccc6dc3f2a05fe762d6d3372cb446d88ac GIT binary patch literal 1397 zcmXqLVl6aiVu@V9%*4pVB#@u9Us#IesM(YUabM(~3ac9MvT?DFX?R2$!&+v%jyui@$=iqk>OnZe~epilL2x6-bU**cd7o6zr(rUzDDhmsyoq zl9`{U5SEyenF3)3rzV#cr78rc7L@_*baph56X!KFH!wCbHUNVtab6<>12ZEdBV$ub zQ_CoWY$SJw1{(?+2twS=Ed4`e(;vm)B!UDdfIVG98F8R5MnRyDq!9E5}j7rD>$H>aS+{DPw02Jq9 zYGPz$IK=YiN6$N!Lz`~|TrG$=Gg;zw%!&-p4N6;oPMx<=Tc^w}=YV&DkI$=3TMqCy zY}@d&=wuty<y8VeR_DBi?&nPSFc6UzRSD*z?WN>8RUfv3B+OC9k>W3I?eu7U#71 zRE9penEisy)$G-j&A)YDMK2k9 zJXb0_en_M$N8^>b*Zo&HTZ^{TMJ*D$lesb6U)Rz-AiH|l^)LeDLwhwS|v73 zy^Xs(Wq8!*E&kp8-JytK%_NSSJ>f0EjjX4i{mTxS;h^4nTwr%`!TU@JM|RiHnx855 zwmOGb*1Vo9<z44-7_GWflnou?CS@pS5qDU+ce3f47#0 zhnCQGX{UoK2C^Upd@N!tB6HsztZwACa?dQ3s1-Dyyl1Jj{cmt0ljUb*{LjL|%*49D zfCr>p7{q5XV1Q^=Wf3zFVdKzdV`ODzXJ&-6m<$3yiWOKq4crZ^*?52oSQs}MGBPnT zvlwU_Xu=dSF^b7%l#~<{Tj}c;gOi?Ka(-@pN?+{ z-k{bX6vj7TLb9iXGJ8r4RN%%kwn;#ppPyV@fMPCC70{=GEV2gDO_F)}y1<~w&Ck=# zOUzBxOG->BE(UuSIU@i|5MV}NWXSxZ>ULmW+UMOr#Oj|(nYf(rnDb6zkyW|Xz_ z$9{(K%Jus_l6IO3g=HA){+awKO)O|fmTT1Z+#7Ryp6LWmIpcR=gMC-d`J0n;Teo@| z6zW||Pht60z4=n>rj4&JFaGB1)RMV*x5}HhJC+@K>piVA&uZReo2JynpBFM}&U!55 z^j+hWs<_`F(Vy8@W!9tBm(mqy|NpbeP?kwtdY#U4)jc=2tozb#`ZwppYdhvmr}>U7 l*UbLhr)aUs$L-SL{YJ0C_h)*Gh(7yrDof|BxSMr#8URDu=Gy=O literal 0 HcmV?d00001 diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 3ae16529d..8368855cd 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -340,14 +340,8 @@ def _get_names_from_cert_or_req(cert_or_req, load_func, typ): def _get_names_from_loaded_cert_or_req(loaded_cert_or_req): - common_name = loaded_cert_or_req.get_subject().CN # pylint: disable=protected-access - sans = acme_crypto_util._pyopenssl_cert_or_req_san(loaded_cert_or_req) - - if common_name is None: - return sans - else: - return [common_name] + [d for d in sans if d != common_name] + return acme_crypto_util._pyopenssl_cert_or_req_all_names(loaded_cert_or_req) def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): From 42638afc7568c161f467a4f6f385d597a6e329d9 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 26 Jan 2018 11:47:25 +0200 Subject: [PATCH 61/98] Drop support for EOL Python 2.6 and 3.3 * Drop support for EOL Python 2.6 * Use more helpful assertIn/NotIn instead of assertTrue/False * Drop support for EOL Python 3.3 * Remove redundant Python 3.3 code * Restore code for RHEL 6 and virtualenv for Py2.7 * Revert pipstrap.py to upstream * Merge py26_packages and non_py26_packages into all_packages * Revert changes to *-auto in root * Update by calling letsencrypt-auto-source/build.py * Revert permissions for pipstrap.py --- .travis.yml | 8 ---- acme/acme/__init__.py | 10 ---- acme/acme/crypto_util.py | 4 +- acme/acme/crypto_util_test.py | 12 ++--- acme/setup.py | 9 ---- certbot-apache/setup.py | 2 - certbot-compatibility-test/setup.py | 2 - certbot-dns-cloudflare/setup.py | 2 - certbot-dns-cloudxns/setup.py | 1 - certbot-dns-digitalocean/setup.py | 2 - certbot-dns-dnsimple/setup.py | 1 - certbot-dns-dnsmadeeasy/setup.py | 1 - certbot-dns-google/setup.py | 2 - certbot-dns-luadns/setup.py | 1 - certbot-dns-nsone/setup.py | 1 - certbot-dns-rfc2136/setup.py | 2 - certbot-dns-route53/setup.py | 2 - certbot-nginx/setup.py | 2 - certbot/log.py | 48 ++++--------------- certbot/main.py | 12 ----- docs/contributing.rst | 2 +- docs/install.rst | 2 +- letsencrypt-auto-source/letsencrypt-auto | 35 ++++---------- .../letsencrypt-auto.template | 35 ++++---------- letsencrypt-auto-source/tests/auto_test.py | 16 +++---- letshelp-certbot/letshelp_certbot/apache.py | 4 +- letshelp-certbot/setup.py | 2 - setup.py | 9 ---- tests/letstest/scripts/test_tox.sh | 8 +--- tests/run_http_server.py | 2 +- tools/install_and_test.sh | 4 -- tox.ini | 41 ++++------------ 32 files changed, 56 insertions(+), 228 deletions(-) diff --git a/.travis.yml b/.travis.yml index 35666d8e6..1077d99d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,18 +25,10 @@ matrix: addons: - python: "2.7" env: TOXENV=lint - - python: "2.6" - env: TOXENV=py26 - sudo: required - services: docker - python: "2.7" env: TOXENV=py27-oldest sudo: required services: docker - - python: "3.3" - env: TOXENV=py33 - sudo: required - services: docker - python: "3.6" env: TOXENV=py36 sudo: required diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index 5850fa955..e8a0b16a8 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -10,13 +10,3 @@ supported version: `draft-ietf-acme-01`_. https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ -import sys -import warnings - -for (major, minor) in [(2, 6), (3, 3)]: - if sys.version_info[:2] == (major, minor): - warnings.warn( - "Python {0}.{1} support will be dropped in the next release of " - "acme. Please upgrade your Python version.".format(major, minor), - DeprecationWarning, - ) #pragma: no cover diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index b8fba0348..78ba41d0f 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -5,7 +5,6 @@ import logging import os import re import socket -import sys import OpenSSL @@ -130,8 +129,7 @@ def probe_sni(name, host, port=443, timeout=300, context = OpenSSL.SSL.Context(method) context.set_timeout(timeout) - socket_kwargs = {} if sys.version_info < (2, 7) else { - 'source_address': source_address} + socket_kwargs = {'source_address': source_address} host_protocol_agnostic = None if host == '::' or host == '0' else host diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf..22a507811 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -170,9 +170,9 @@ class MakeCSRTest(unittest.TestCase): self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 1) self.assertEquals(csr.get_extensions()[0].get_data(), @@ -188,9 +188,9 @@ class MakeCSRTest(unittest.TestCase): csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 2) # NOTE: Ideally we would filter by the TLS Feature OID, but diff --git a/acme/setup.py b/acme/setup.py index ce426cf74..ba5c8e6fb 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -25,13 +25,6 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ 'pytest', 'pytest-xdist', @@ -58,10 +51,8 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 38f41e9f1..d7c223a0a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 8f9f897cf..7e1b059e2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -40,10 +40,8 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 612e7259f..d619f1872 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -39,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3157400c6..5d14f3e29 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1a68400fa..ce8fedd46 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 35de47308..06af16759 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a946d00a4..7c0f3ed86 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8585fc848..de881ad84 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -44,10 +44,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4fec37e29..0d580b7ee 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index dca9ebf27..c0ba11470 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bfa72b50b..5161e7a94 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -39,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8df687972..09f8a7d52 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -32,10 +32,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 152f77de8..37c477ef6 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot/log.py b/certbot/log.py index f7c7b126c..e0d2e8f11 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -165,12 +165,7 @@ class ColoredStreamHandler(logging.StreamHandler): """ def __init__(self, stream=None): - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(ColoredStreamHandler, self).__init__(stream) + super(ColoredStreamHandler, self).__init__(stream) self.colored = (sys.stderr.isatty() if stream is None else stream.isatty()) self.red_level = logging.WARNING @@ -184,9 +179,7 @@ class ColoredStreamHandler(logging.StreamHandler): :rtype: str """ - out = (logging.StreamHandler.format(self, record) - if sys.version_info < (2, 7) - else super(ColoredStreamHandler, self).format(record)) + out = super(ColoredStreamHandler, self).format(record) if self.colored and record.levelno >= self.red_level: return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) else: @@ -203,23 +196,14 @@ class MemoryHandler(logging.handlers.MemoryHandler): def __init__(self, target=None): # capacity doesn't matter because should_flush() is overridden capacity = float('inf') - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.__init__( - self, capacity, target=target) - else: - super(MemoryHandler, self).__init__(capacity, target=target) + super(MemoryHandler, self).__init__(capacity, target=target) def close(self): """Close the memory handler, but don't set the target to None.""" # This allows the logging module which may only have a weak # reference to the target handler to properly flush and close it. target = self.target - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.close(self) - else: - super(MemoryHandler, self).close() + super(MemoryHandler, self).close() self.target = target def flush(self, force=False): # pylint: disable=arguments-differ @@ -233,10 +217,7 @@ class MemoryHandler(logging.handlers.MemoryHandler): # This method allows flush() calls in logging.shutdown to be a # noop so we can control when this handler is flushed. if force: - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.flush(self) - else: - super(MemoryHandler, self).flush() + super(MemoryHandler, self).flush() def shouldFlush(self, record): """Should the buffer be automatically flushed? @@ -262,12 +243,7 @@ class TempHandler(logging.StreamHandler): """ def __init__(self): stream = tempfile.NamedTemporaryFile('w', delete=False) - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(TempHandler, self).__init__(stream) + super(TempHandler, self).__init__(stream) self.path = stream.name self._delete = True @@ -278,12 +254,7 @@ class TempHandler(logging.StreamHandler): """ self._delete = False - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.emit(self, record) - else: - super(TempHandler, self).emit(record) + super(TempHandler, self).emit(record) def close(self): """Close the handler and the temporary log file. @@ -299,10 +270,7 @@ class TempHandler(logging.StreamHandler): if self._delete: os.remove(self.path) self._delete = False - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.close(self) - else: - super(TempHandler, self).close() + super(TempHandler, self).close() finally: self.release() diff --git a/certbot/main.py b/certbot/main.py index 32dd69256..76a90d499 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -4,7 +4,6 @@ import functools import logging.handlers import os import sys -import warnings import configobj import josepy as jose @@ -1218,17 +1217,6 @@ def main(cli_args=sys.argv[1:]): # Let plugins_cmd be run as un-privileged user. if config.func != plugins_cmd: raise - deprecation_fmt = ( - "Python %s.%s support will be dropped in the next " - "release of Certbot - please upgrade your Python version.") - # We use the warnings system for Python 2.6 and logging for Python 3 - # because DeprecationWarnings are only reported by default in Python <= 2.6 - # and warnings can be disabled by the user. - if sys.version_info[:2] == (2, 6): - warning = deprecation_fmt % sys.version_info[:2] - warnings.warn(warning, DeprecationWarning) - elif sys.version_info[:2] == (3, 3): - logger.warning(deprecation_fmt, *sys.version_info[:2]) set_displayer(config) diff --git a/docs/contributing.rst b/docs/contributing.rst index 83b607e15..654528e3d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -422,7 +422,7 @@ OS-level dependencies can be installed like so: In general... * ``sudo`` is required as a suggested way of running privileged process -* `Python`_ 2.6/2.7 is required +* `Python`_ 2.7 is required * `Augeas`_ is required for the Python bindings * ``virtualenv`` and ``pip`` are used for managing other python library dependencies diff --git a/docs/install.rst b/docs/install.rst index c18c3cdbc..aec885b62 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,7 +22,7 @@ your system. System Requirements =================== -Certbot currently requires Python 2.6, 2.7, or 3.3+. By default, it requires +Certbot currently requires Python 2.7, or 3.4+. By default, it requires root access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and modify webserver diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8ff7944b5..aed15a8ef 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -246,7 +246,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -781,20 +781,11 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -965,18 +956,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2ce337002..b3d6ab740 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -246,7 +246,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -320,20 +320,11 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -504,18 +495,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index d187452a1..8c2bfc079 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -287,8 +287,8 @@ class AutoTests(TestCase): self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: - self.assertTrue('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This # conveniently sets us up to test the next 2 cases. @@ -296,8 +296,8 @@ class AutoTests(TestCase): # Test when neither phase-1 upgrade nor phase-2 upgrade is # needed (probably a common case): out, err = run_letsencrypt_auto() - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertFalse('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertNotIn('Creating virtual environment...', out) def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" @@ -312,8 +312,8 @@ class AutoTests(TestCase): # Create venv saving the correct bootstrap script version out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: bootstrap_version = f.read() @@ -329,8 +329,8 @@ class AutoTests(TestCase): out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) def test_openssl_failure(self): """Make sure we stop if the openssl signature check fails.""" diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index b13057ca5..f77a6a1b0 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -5,7 +5,6 @@ from __future__ import print_function import argparse import atexit -import contextlib import os import re import shutil @@ -302,8 +301,7 @@ def main(): make_and_verify_selection(args.server_root, tempdir) tarpath = os.path.join(tempdir, "config.tar.gz") - # contextlib.closing used for py26 support - with contextlib.closing(tarfile.open(tarpath, mode="w:gz")) as tar: + with tarfile.open(tarpath, mode="w:gz") as tar: tar.add(tempdir, arcname=".") # TODO: Submit tarpath diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 3ce442b3e..7c8c39068 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -31,10 +31,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/setup.py b/setup.py index ce505a62e..e3824a7f7 100644 --- a/setup.py +++ b/setup.py @@ -52,13 +52,6 @@ install_requires = [ 'zope.interface', ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', @@ -98,10 +91,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index 4c2eb429e..84e4bcd22 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -15,10 +15,4 @@ VENV_BIN=${VENV_PATH}/bin cd letsencrypt ./tools/venv.sh -PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - -if [ $PYVER -eq 26 ] ; then - venv/bin/tox -e py26 -else - venv/bin/tox -e py27 -fi +venv/bin/tox -e py27 diff --git a/tests/run_http_server.py b/tests/run_http_server.py index fd1163816..0e4f8ac79 100644 --- a/tests/run_http_server.py +++ b/tests/run_http_server.py @@ -3,7 +3,7 @@ import sys # Run Python's built-in HTTP server # Usage: python ./tests/run_http_server.py port_num -# NOTE: This script should be compatible with 2.6, 2.7, 3.3+ +# NOTE: This script should be compatible with 2.7, 3.4+ # sys.argv (port number) is passed as-is to the HTTP server module runpy.run_module( diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 0d39e0594..f0385470b 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -18,10 +18,6 @@ for requirement in "$@" ; do pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] if [ $pkg = "." ]; then pkg="certbot" - else - # Work around a bug in pytest/importlib for the deprecated Python 3.3. - # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. - pkg=$(echo "$pkg" | tr - _) fi "$(dirname $0)/pytest.sh" --pyargs $pkg done diff --git a/tox.ini b/tox.ini index 20f5cda32..971aa7631 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py{26,33,34,35,36},cover,lint +envlist = modification,py{34,35,36},cover,lint [base] # pip installs the requested packages in editable mode @@ -14,25 +14,22 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -py26_packages = +all_packages = acme[dev] \ .[dev] \ certbot-apache \ certbot-dns-cloudflare \ + certbot-dns-cloudxns \ certbot-dns-digitalocean \ + certbot-dns-dnsimple \ + certbot-dns-dnsmadeeasy \ certbot-dns-google \ + certbot-dns-luadns \ + certbot-dns-nsone \ certbot-dns-rfc2136 \ certbot-dns-route53 \ certbot-nginx \ letshelp-certbot -non_py26_packages = - certbot-dns-cloudxns \ - certbot-dns-dnsimple \ - certbot-dns-dnsmadeeasy \ - certbot-dns-luadns \ - certbot-dns-nsone -all_packages = - {[base]py26_packages} {[base]non_py26_packages} install_packages = {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} source_paths = @@ -54,32 +51,15 @@ source_paths = letshelp-certbot/letshelp_certbot tests/lock_test.py -[testenv:py26] -commands = - {[base]install_and_test} {[base]py26_packages} - python tests/lock_test.py -deps = - setuptools==36.8.0 - wheel==0.29.0 -passenv = TRAVIS - [testenv] commands = - {[testenv:py26]commands} - {[base]install_and_test} {[base]non_py26_packages} + {[base]install_and_test} {[base]all_packages} + python tests/lock_test.py setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 passenv = - {[testenv:py26]passenv} - -[testenv:py33] -commands = - {[testenv]commands} -deps = - wheel==0.29.0 -passenv = - {[testenv]passenv} + TRAVIS [testenv:py27-oldest] commands = @@ -104,7 +84,6 @@ passenv = {[testenv]passenv} [testenv:lint] -# recent versions of pylint do not support Python 2.6 (#97, #187) basepython = python2.7 # separating into multiple invocations disables cross package # duplicate code checking; if one of the commands fails, others will From 73bd801f352fb9bad7fe8bc35c368f573c15a21e Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:21:02 -0800 Subject: [PATCH 62/98] add and use request_authorizations --- acme/acme/client.py | 14 ++++++++++++++ certbot/auth_handler.py | 10 ++++++---- certbot/client.py | 23 ++++++++--------------- certbot/main.py | 2 +- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 1f4ae4fad..6b4d65233 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -671,6 +671,7 @@ class BackwardsCompatibleClientV2(object): self.client = Client(directory, key=key, net=net) else: self.client = ClientV2(directory, net=net) + self.orderr = None def __getattr__(self, name): if name in vars(self.client): @@ -705,6 +706,19 @@ class BackwardsCompatibleClientV2(object): regr = regr.update(terms_of_service_agreed=True) return self.client.new_account(regr) + def request_authorizations(self, csr_pem): + if self.acme_version == 1: + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) + authorizations = [] + for domain in dnsNames: + authorizations.append(self.client.request_domain_challenges(domain)) + return authorizations + else: + self.orderr = self.client.new_order(csr_pem) + return self.orderr.authorizations + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 5f520cbcb..4f88199e3 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,10 +48,10 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def get_authorizations(self, domains, best_effort=False): + def get_authorizations(self, csr_pem, best_effort=False): """Retrieve all authorizations for challenges. - :param list domains: Domains for authorization + :param list csr_pem: CSR containing domains for authorization :param bool best_effort: Whether or not all authorizations are required (this is useful in renewal) @@ -62,8 +62,10 @@ class AuthHandler(object): authorizations """ - for domain in domains: - self.authzr[domain] = self.acme.request_domain_challenges(domain) + authzrs = self.acme.request_authorizations(csr_pem) + for authzr in authzrs: + self.authzr[authzr.body.identifier.value] = authzr + domains = self.authzr.keys() self._choose_challenges(domains) config = zope.component.getUtility(interfaces.IConfig) diff --git a/certbot/client.py b/certbot/client.py index 67ee8f7fa..8e3ec6c62 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,13 +235,9 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, domains, csr, authzr=None): + def obtain_certificate_from_csr(self, csr, authzr=None): """Obtain certificate. - Internal function with precondition that `domains` are - consistent with identifiers present in the `csr`. - - :param list domains: Domain names. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. @@ -261,10 +257,10 @@ class Client(object): if self.account.regr is None: raise errors.Error("Please register with the ACME server first.") - logger.debug("CSR: %s, domains: %s", csr, domains) + logger.debug("CSR: %s", csr) if authzr is None: - authzr = self.auth_handler.get_authorizations(domains) + authzr = self.auth_handler.get_authorizations(csr) certr = self.acme.request_issuance( jose.ComparableX509( @@ -307,13 +303,6 @@ class Client(object): :rtype: tuple """ - authzr = self.auth_handler.get_authorizations( - domains, - self.config.allow_subset_of_names) - - auth_domains = set(a.body.identifier.value for a in authzr) - domains = [d for d in domains if d in auth_domains] - # Create CSR from names if self.config.dry_run: key = util.Key(file=None, @@ -326,8 +315,12 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) + authzr = self.auth_handler.get_authorizations( + csr, + self.config.allow_subset_of_names) + certr, chain = self.obtain_certificate_from_csr( - domains, csr, authzr=authzr) + csr, authzr=authzr) return certr, chain, key, csr diff --git a/certbot/main.py b/certbot/main.py index ff3758985..d01f68920 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1064,7 +1064,7 @@ def _csr_get_and_save_cert(config, le_client): """ csr, _ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr) + certr, chain = le_client.obtain_certificate_from_csr(csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) From eaf739184cf517d5a3f5103caa072bb0cd39b4e9 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:29:42 -0800 Subject: [PATCH 63/98] pass pem to auth_handler --- certbot/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 8e3ec6c62..d7d2acb14 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -260,7 +260,7 @@ class Client(object): logger.debug("CSR: %s", csr) if authzr is None: - authzr = self.auth_handler.get_authorizations(csr) + authzr = self.auth_handler.get_authorizations(csr.data) certr = self.acme.request_issuance( jose.ComparableX509( @@ -316,7 +316,7 @@ class Client(object): csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) authzr = self.auth_handler.get_authorizations( - csr, + csr.data, self.config.allow_subset_of_names) certr, chain = self.obtain_certificate_from_csr( From ea2022588b4d95f1d14849b918b2cd3788cc6084 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:32:49 -0800 Subject: [PATCH 64/98] add docstring --- acme/acme/client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/acme/acme/client.py b/acme/acme/client.py index 6b4d65233..e7cf016bb 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -707,6 +707,16 @@ class BackwardsCompatibleClientV2(object): return self.client.new_account(regr) def request_authorizations(self, csr_pem): + """Request authorizations for the domains in csr_pem. + + Calls request_domain_challenges for each domain for V1, and + calls new_order and saves the result for V2. + + :param str csr_pem: A CSR in PEM format. + + :returns: List of Authorization Resources. + :rtype: list of `.AuthorizationResource` + """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) # pylint: disable=protected-access From 20d0b91c710bd8110aa5ee23082f45583e7789a4 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 17:35:10 -0800 Subject: [PATCH 65/98] switch interface to new_order and remove best_effort flag --- acme/acme/client.py | 18 ++++++++---------- certbot/auth_handler.py | 27 ++++++++++----------------- certbot/client.py | 15 ++++----------- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index e7cf016bb..1838fab42 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -671,7 +671,6 @@ class BackwardsCompatibleClientV2(object): self.client = Client(directory, key=key, net=net) else: self.client = ClientV2(directory, net=net) - self.orderr = None def __getattr__(self, name): if name in vars(self.client): @@ -706,16 +705,16 @@ class BackwardsCompatibleClientV2(object): regr = regr.update(terms_of_service_agreed=True) return self.client.new_account(regr) - def request_authorizations(self, csr_pem): - """Request authorizations for the domains in csr_pem. + def new_order(self, csr_pem): + """Request a new Order object from the server. - Calls request_domain_challenges for each domain for V1, and - calls new_order and saves the result for V2. + If using ACMEv1, returns a dummy OrderResource with only + the authorizations field filled in. :param str csr_pem: A CSR in PEM format. - :returns: List of Authorization Resources. - :rtype: list of `.AuthorizationResource` + :returns: The newly created order. + :rtype: OrderResource """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) @@ -724,10 +723,9 @@ class BackwardsCompatibleClientV2(object): authorizations = [] for domain in dnsNames: authorizations.append(self.client.request_domain_challenges(domain)) - return authorizations + return messages.OrderResource(authorizations=authorizations) else: - self.orderr = self.client.new_order(csr_pem) - return self.orderr.authorizations + return self.client.new_order(csr_pem) def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 4f88199e3..825513329 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,12 +48,11 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def get_authorizations(self, csr_pem, best_effort=False): + def handle_authorizations(self, orderr): """Retrieve all authorizations for challenges. - :param list csr_pem: CSR containing domains for authorization - :param bool best_effort: Whether or not all authorizations are - required (this is useful in renewal) + :param acme.messages.OrderResource orderr: must have + authorizations filled in :returns: List of authorization resources :rtype: list @@ -62,7 +61,7 @@ class AuthHandler(object): authorizations """ - authzrs = self.acme.request_authorizations(csr_pem) + authzrs = orderr.authorizations for authzr in authzrs: self.authzr[authzr.body.identifier.value] = authzr domains = self.authzr.keys() @@ -80,7 +79,7 @@ class AuthHandler(object): 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp, best_effort) + self._respond(resp) # Just make sure all decisions are complete. self.verify_authzr_complete() @@ -124,7 +123,7 @@ class AuthHandler(object): return resp - def _respond(self, resp, best_effort): + def _respond(self, resp): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -137,7 +136,7 @@ class AuthHandler(object): # Check for updated status... try: - self._poll_challenges(chall_update, best_effort) + self._poll_challenges(chall_update) finally: # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) @@ -169,7 +168,7 @@ class AuthHandler(object): return active_achalls def _poll_challenges( - self, chall_update, best_effort, min_sleep=3, max_rounds=15): + self, chall_update, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" dom_to_check = set(chall_update.keys()) comp_domains = set() @@ -190,14 +189,8 @@ class AuthHandler(object): chall_update[domain].remove(achall) # We failed some challenges... damage control else: - if best_effort: - comp_domains.add(domain) - logger.warning( - "Challenge failed for domain %s", - domain) - else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) + all_failed_achalls.update( + updated for _, updated in failed_achalls) if all_failed_achalls: _report_failed_challs(all_failed_achalls) diff --git a/certbot/client.py b/certbot/client.py index d7d2acb14..61e9db635 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,14 +235,12 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr, authzr=None): + def obtain_certificate_from_csr(self, csr): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. - :param list authzr: List of - :class:`acme.messages.AuthorizationResource` :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -259,8 +257,8 @@ class Client(object): logger.debug("CSR: %s", csr) - if authzr is None: - authzr = self.auth_handler.get_authorizations(csr.data) + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr) certr = self.acme.request_issuance( jose.ComparableX509( @@ -315,12 +313,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - authzr = self.auth_handler.get_authorizations( - csr.data, - self.config.allow_subset_of_names) - - certr, chain = self.obtain_certificate_from_csr( - csr, authzr=authzr) + certr, chain = self.obtain_certificate_from_csr(csr) return certr, chain, key, csr From 68e24a8ea7eeb405592e40ed340baff6f3f3821b Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 17:59:51 -0800 Subject: [PATCH 66/98] start test updates --- certbot/tests/auth_handler_test.py | 54 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 32c4c0d3b..d424e59ca 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -57,8 +57,8 @@ class ChallengeFactoryTest(unittest.TestCase): errors.Error, self.handler._challenge_factory, "failure.com", [0]) -class GetAuthorizationsTest(unittest.TestCase): - """get_authorizations test. +class HandleAuthorizationsTest(unittest.TestCase): + """handle_authorizations test. This tests everything except for all functions under _poll_challenges. @@ -92,12 +92,11 @@ class GetAuthorizationsTest(unittest.TestCase): @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_name1_tls_sni_01_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) - mock_poll.side_effect = self._validate_all - authzr = self.handler.get_authorizations(["0"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) @@ -115,14 +114,13 @@ class GetAuthorizationsTest(unittest.TestCase): @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES, combos=False) - mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = self.handler.get_authorizations(["0"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -146,7 +144,11 @@ class GetAuthorizationsTest(unittest.TestCase): mock_poll.side_effect = self._validate_all - authzr = self.handler.get_authorizations(["0", "1", "2"]) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -169,31 +171,33 @@ class GetAuthorizationsTest(unittest.TestCase): def test_debug_challenges(self, mock_poll): zope.component.provideUtility( mock.Mock(debug_challenges=True), interfaces.IConfig) - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) mock_poll.side_effect = self._validate_all - self.handler.get_authorizations(["0"]) + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(self.mock_display.notification.call_count, 1) def test_perform_failure(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + self.mock_auth.perform.side_effect = errors.AuthorizationError self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) def test_no_domains(self): - self.assertRaises(errors.AuthorizationError, self.handler.get_authorizations, []) + mock_order = mock.MagicMock(authorizations=[]) + self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_preferred_challenge_choice(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) @@ -201,20 +205,20 @@ class GetAuthorizationsTest(unittest.TestCase): self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - self.handler.get_authorizations(["0"]) + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") def test_preferred_challenges_not_supported(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) self.handler.pref_challs.append(challenges.HTTP01.typ) self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1, unused_2): + def _validate_all(self, unused_1): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] self.handler.authzr[dom] = acme_util.gen_authzr( From 9c84fe1144cc5ccec5fc9cfcdf7c5adf8cfdf15c Mon Sep 17 00:00:00 2001 From: Matt Christian Date: Sun, 18 Feb 2018 15:45:22 -0600 Subject: [PATCH 67/98] Add override class for ID="ol" AKA Oracle Linux Server, a clone of CentOS/RHEL. --- certbot-apache/certbot_apache/entrypoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-apache/certbot_apache/entrypoint.py b/certbot-apache/certbot_apache/entrypoint.py index 4267398d5..6f1443507 100644 --- a/certbot-apache/certbot_apache/entrypoint.py +++ b/certbot-apache/certbot_apache/entrypoint.py @@ -17,6 +17,7 @@ OVERRIDE_CLASSES = { "centos": override_centos.CentOSConfigurator, "centos linux": override_centos.CentOSConfigurator, "fedora": override_centos.CentOSConfigurator, + "ol": override_centos.CentOSConfigurator, "red hat enterprise linux server": override_centos.CentOSConfigurator, "rhel": override_centos.CentOSConfigurator, "amazon": override_centos.CentOSConfigurator, From d6b4e2001b404b1f9bd4c3929e888726d583ee57 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 13:19:04 -0800 Subject: [PATCH 68/98] put back in best_effort code, with a todo for actually supporting it in ACMEv2 --- certbot/auth_handler.py | 22 +++++++++++++++------- certbot/client.py | 11 ++++++++--- certbot/tests/auth_handler_test.py | 2 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 825513329..662cadc65 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,11 +48,13 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def handle_authorizations(self, orderr): + def handle_authorizations(self, orderr, best_effort=False): """Retrieve all authorizations for challenges. :param acme.messages.OrderResource orderr: must have authorizations filled in + :param bool best_effort: Whether or not all authorizations are + required (this is useful in renewal) :returns: List of authorization resources :rtype: list @@ -79,7 +81,7 @@ class AuthHandler(object): 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp) + self._respond(resp, best_effort) # Just make sure all decisions are complete. self.verify_authzr_complete() @@ -123,7 +125,7 @@ class AuthHandler(object): return resp - def _respond(self, resp): + def _respond(self, resp, best_effort): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -136,7 +138,7 @@ class AuthHandler(object): # Check for updated status... try: - self._poll_challenges(chall_update) + self._poll_challenges(chall_update, best_effort) finally: # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) @@ -168,7 +170,7 @@ class AuthHandler(object): return active_achalls def _poll_challenges( - self, chall_update, min_sleep=3, max_rounds=15): + self, chall_update, best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" dom_to_check = set(chall_update.keys()) comp_domains = set() @@ -189,8 +191,14 @@ class AuthHandler(object): chall_update[domain].remove(achall) # We failed some challenges... damage control else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) + if best_effort: + comp_domains.add(domain) + logger.warning( + "Challenge failed for domain %s", + domain) + else: + all_failed_achalls.update( + updated for _, updated in failed_achalls) if all_failed_achalls: _report_failed_challs(all_failed_achalls) diff --git a/certbot/client.py b/certbot/client.py index 61e9db635..5feea662d 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,12 +235,13 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr): + def obtain_certificate_from_csr(self, csr, best_effort=False): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. + :param bool best_effort: Whether or not all authorizations are required :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -258,7 +259,11 @@ class Client(object): logger.debug("CSR: %s", csr) orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr) + authzr = self.auth_handler.handle_authorizations(orderr, best_effort) + if best_effort: + # TODO: check if we passed all authorizations, and if not, + # create a new order and try again, possibly in a loop + pass certr = self.acme.request_issuance( jose.ComparableX509( @@ -313,7 +318,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - certr, chain = self.obtain_certificate_from_csr(csr) + certr, chain = self.obtain_certificate_from_csr(csr, self.config.allow_subset_of_names) return certr, chain, key, csr diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index d424e59ca..ea8b006c4 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -218,7 +218,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1): + def _validate_all(self, unused_1, unused_2): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] self.handler.authzr[dom] = acme_util.gen_authzr( From 11f2f1e576243a255afbcd166b042cab2e2f1c4b Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 13:20:41 -0800 Subject: [PATCH 69/98] remove extra spaces --- certbot/auth_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 662cadc65..47d806b94 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -194,8 +194,8 @@ class AuthHandler(object): if best_effort: comp_domains.add(domain) logger.warning( - "Challenge failed for domain %s", - domain) + "Challenge failed for domain %s", + domain) else: all_failed_achalls.update( updated for _, updated in failed_achalls) From a0e84e65ce9cfc041f341a6bedf36525e49fa1b2 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:29:04 -0800 Subject: [PATCH 70/98] auth_handler tests are happy --- certbot/tests/auth_handler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index ea8b006c4..3633b673d 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -118,7 +118,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) mock_order = mock.MagicMock(authorizations=[authzr]) authzr = self.handler.handle_authorizations(mock_order) From 76a0cbf9c23b9827c4bf94e9a6521a4bf049a466 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:43:12 -0800 Subject: [PATCH 71/98] client tests passing --- certbot/tests/client_test.py | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a9a87b80b..b6cbca367 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -141,16 +141,17 @@ class ClientTest(ClientTestCommon): def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() - self.client.auth_handler.get_authorizations.return_value = [None] + self.client.auth_handler.handle_authorizations.return_value = [None] self.acme.request_issuance.return_value = mock.sentinel.certr self.acme.fetch_chain.return_value = mock.sentinel.chain + self.acme.new_order.return_value = mock.sentinel.orderr def _check_obtain_certificate(self): - self.client.auth_handler.get_authorizations.assert_called_once_with( - self.eg_domains, + self.client.auth_handler.handle_authorizations.assert_called_once_with( + mock.sentinel.orderr, self.config.allow_subset_of_names) - authzr = self.client.auth_handler.get_authorizations() + authzr = self.client.auth_handler.handle_authorizations() self.acme.request_issuance.assert_called_once_with( jose.ComparableX509(OpenSSL.crypto.load_certificate_request( @@ -167,31 +168,19 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, test_csr, - authzr=authzr)) + best_effort=False)) # and that the cert was obtained correctly self._check_obtain_certificate() - # Test for authzr=None - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=None)) - auth_handler.get_authorizations.assert_called_with(self.eg_domains) - # Test for no auth_handler self.client.auth_handler = None self.assertRaises( errors.Error, self.client.obtain_certificate_from_csr, - self.eg_domains, test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) @@ -204,13 +193,10 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="der", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=authzr)) + test_csr)) self.assertEqual(1, mock_get_utility().notification.call_count) @test_util.patch_get_utility() @@ -220,13 +206,10 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="der", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, - self.eg_domains, - test_csr, - authzr=authzr) + test_csr) self.assertEqual(1, mock_get_utility().notification.call_count) @mock.patch("certbot.client.crypto_util") @@ -276,7 +259,7 @@ class ClientTest(ClientTestCommon): identifier=mock.MagicMock( value=domain)))) - self.client.auth_handler.get_authorizations.return_value = authzr + self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): result = self.client.obtain_certificate(self.eg_domains) From 3dfeb483ee853333b11e829ef7d985ebf5ad7269 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:49:23 -0800 Subject: [PATCH 72/98] lint --- certbot/tests/client_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index b6cbca367..570080e3b 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -166,7 +166,6 @@ class ClientTest(ClientTestCommon): mock_logger): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), @@ -191,7 +190,6 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), @@ -204,7 +202,6 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertRaises( acme_errors.Error, From d6af978472d7519b615e8894903383610ab41269 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:52:11 -0800 Subject: [PATCH 73/98] remove if/pass --- certbot/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 5feea662d..65e85a159 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -260,10 +260,8 @@ class Client(object): orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr, best_effort) - if best_effort: - # TODO: check if we passed all authorizations, and if not, - # create a new order and try again, possibly in a loop - pass + # TODO: check if we passed all authorizations, and if not, + # create a new order and try again, possibly in a loop certr = self.acme.request_issuance( jose.ComparableX509( From d29c637bf94cf083ef3dc1648bed3697229b8025 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:36:35 -0800 Subject: [PATCH 74/98] support best_effort --- certbot/client.py | 29 +++++++++++++++++++++-------- certbot/tests/client_test.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 65e85a159..d00055eae 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,13 +235,13 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr, best_effort=False): + def obtain_certificate_from_csr(self, csr, orderr=None): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. - :param bool best_effort: Whether or not all authorizations are required + :param acme.messages.OrderResource orderr: contains authzrs :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -258,10 +258,12 @@ class Client(object): logger.debug("CSR: %s", csr) - orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr, best_effort) - # TODO: check if we passed all authorizations, and if not, - # create a new order and try again, possibly in a loop + if orderr is None: + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr) + else: + authzr = orderr.authorizations + certr = self.acme.request_issuance( jose.ComparableX509( @@ -316,9 +318,20 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - certr, chain = self.obtain_certificate_from_csr(csr, self.config.allow_subset_of_names) + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) + auth_domains = set(a.body.identifier.value for a in authzr) + successful_domains = [d for d in domains if d in auth_domains] - return certr, chain, key, csr + if successful_domains != domains: + if not self.config.dry_run: + # TODO: delete keys + pass + return self.obtain_certificate(successful_domains) + else: + certr, chain = self.obtain_certificate_from_csr(csr, orderr) + + return certr, chain, key, csr # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 570080e3b..376bf5a90 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -134,6 +134,7 @@ class ClientTest(ClientTestCommon): self.config.allow_subset_of_names = False self.config.dry_run = False self.eg_domains = ["example.com", "www.example.com"] + self.eg_order = mock.MagicMock(authorizations=[None]) def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] @@ -144,11 +145,11 @@ class ClientTest(ClientTestCommon): self.client.auth_handler.handle_authorizations.return_value = [None] self.acme.request_issuance.return_value = mock.sentinel.certr self.acme.fetch_chain.return_value = mock.sentinel.chain - self.acme.new_order.return_value = mock.sentinel.orderr + self.acme.new_order.return_value = self.eg_order def _check_obtain_certificate(self): self.client.auth_handler.handle_authorizations.assert_called_once_with( - mock.sentinel.orderr, + self.eg_order, self.config.allow_subset_of_names) authzr = self.client.auth_handler.handle_authorizations() @@ -166,15 +167,26 @@ class ClientTest(ClientTestCommon): mock_logger): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, - best_effort=False)) + orderr=orderr)) # and that the cert was obtained correctly self._check_obtain_certificate() + # Test for orderr=None + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr( + test_csr, + orderr=None)) + auth_handler.handle_authorizations.assert_called_with(self.eg_order) + # Test for no auth_handler self.client.auth_handler = None self.assertRaises( @@ -190,11 +202,15 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - test_csr)) + test_csr, + orderr=orderr)) self.assertEqual(1, mock_get_utility().notification.call_count) @test_util.patch_get_utility() @@ -202,11 +218,15 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, - test_csr) + test_csr, + orderr=orderr) self.assertEqual(1, mock_get_utility().notification.call_count) @mock.patch("certbot.client.crypto_util") @@ -256,6 +276,7 @@ class ClientTest(ClientTestCommon): identifier=mock.MagicMock( value=domain)))) + self.eg_order.authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): From 7c073dbcaf6d6789da01734f0a32951b18b3e4d6 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:38:18 -0800 Subject: [PATCH 75/98] lint --- certbot/tests/client_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 376bf5a90..f0ef077b2 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,7 +170,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( @@ -205,7 +204,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( @@ -221,7 +219,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, From 051664a142a8c77121c79cb07307379a63d76874 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:39:30 -0800 Subject: [PATCH 76/98] lint --- certbot/tests/client_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f0ef077b2..f10053616 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -201,7 +201,6 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) self.assertEqual( @@ -216,7 +215,6 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) self.assertRaises( From d5a90c5a6e58a068c0fbd8bf800f9953425a9693 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:43:27 -0800 Subject: [PATCH 77/98] delete key and csr before trying again --- certbot/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index d00055eae..404e1e0d9 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -325,8 +325,8 @@ class Client(object): if successful_domains != domains: if not self.config.dry_run: - # TODO: delete keys - pass + os.remove(key.file) + os.remove(csr.file) return self.obtain_certificate(successful_domains) else: certr, chain = self.obtain_certificate_from_csr(csr, orderr) From 26bcaff85cae0b38280f576ac732bd5a4744d2f8 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:59:58 -0800 Subject: [PATCH 78/98] add test for new_order for v2 --- acme/acme/client_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 11516c02f..1ba41cd7d 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -161,6 +161,21 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().register.assert_called_once_with(self.new_reg) mock_client().agree_to_tos.assert_not_called() + def test_new_order_v1(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + + def test_new_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_csr_pem = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_order(mock_csr_pem) + mock_client().new_order.assert_called_once_with(mock_csr_pem) + + + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" From 65d0b9674cb4008e624c1a58cc7c8e4ee91fcdb6 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:01:35 -0800 Subject: [PATCH 79/98] Fix client test --- certbot/tests/client_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f10053616..6b90eab83 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,6 +170,7 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) + auth_handler.handle_authorizations(orderr) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( From a7eadf88629d9cbba611731b9fe555bc0ba5cb84 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:08:46 -0800 Subject: [PATCH 80/98] add new order test for v1 --- acme/acme/client_test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1ba41cd7d..773d59aa2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -161,10 +161,18 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().register.assert_called_once_with(self.new_reg) mock_client().agree_to_tos.assert_not_called() - def test_new_order_v1(self): + @mock.patch('OpenSSL.crypto.load_certificate_request') + @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names') + def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names, + mock_load_certificate_request): self.response.json.return_value = DIRECTORY_V1.to_json() + mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com'] + mock_csr_pem = mock.MagicMock() with mock.patch('acme.client.Client') as mock_client: + mock_client().request_domain_challenges.return_value = mock.sentinel.auth client = self._init() + orderr = client.new_order(mock_csr_pem) + self.assertEqual(orderr.authorizations, [mock.sentinel.auth, mock.sentinel.auth]) def test_new_order_v2(self): self.response.json.return_value = DIRECTORY_V2.to_json() From dea43e90b629f629a19e6f865fbac2930421a733 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:11:36 -0800 Subject: [PATCH 81/98] lint --- acme/acme/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 773d59aa2..1b33ca5d7 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -164,7 +164,7 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): @mock.patch('OpenSSL.crypto.load_certificate_request') @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names') def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names, - mock_load_certificate_request): + unused_mock_load_certificate_request): self.response.json.return_value = DIRECTORY_V1.to_json() mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com'] mock_csr_pem = mock.MagicMock() From df50f2d5fa3ddd70cfcd01cf63c305431c577d46 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:12:15 -0800 Subject: [PATCH 82/98] client test --- certbot/tests/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 6b90eab83..ecd77bdeb 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,7 +170,7 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - auth_handler.handle_authorizations(orderr) + auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( From d13a4ed18da3f2a0b8f88076bf15f778337259ea Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:50:18 -0800 Subject: [PATCH 83/98] add tests for if partial auth success --- certbot/tests/client_test.py | 47 ++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index ecd77bdeb..f4a8a5c8a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -147,10 +147,13 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.return_value = mock.sentinel.chain self.acme.new_order.return_value = self.eg_order - def _check_obtain_certificate(self): - self.client.auth_handler.handle_authorizations.assert_called_once_with( - self.eg_order, - self.config.allow_subset_of_names) + def _check_obtain_certificate(self, auth_count=1): + if auth_count == 1: + self.client.auth_handler.handle_authorizations.assert_called_once_with( + self.eg_order, + self.config.allow_subset_of_names) + else: + self.assertEqual(self.client.auth_handler.handle_authorizations.call_count, auth_count) authzr = self.client.auth_handler.handle_authorizations() @@ -238,6 +241,21 @@ class ClientTest(ClientTestCommon): mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) + @mock.patch("certbot.client.crypto_util") + @mock.patch("os.remove") + def test_obtain_certificate_partial_success(self, mock_remove, mock_crypto_util): + csr = util.CSR(form="pem", file=mock.sentinel.csr_file, data=CSR_SAN) + key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) + mock_crypto_util.init_save_csr.return_value = csr + mock_crypto_util.init_save_key.return_value = key + + authzr = self._authzr_from_domains(["example.com"]) + self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) + + self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) + self.assertEqual(mock_crypto_util.init_save_csr.call_count, 2) + self.assertEqual(mock_remove.call_count, 2) + @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.acme_crypto_util") def test_obtain_certificate_dry_run(self, mock_acme_crypto, mock_crypto): @@ -255,22 +273,25 @@ class ClientTest(ClientTestCommon): mock_crypto.init_save_key.assert_not_called() mock_crypto.init_save_csr.assert_not_called() - def _test_obtain_certificate_common(self, key, csr): - self._mock_obtain_certificate() - - # return_value is essentially set to (None, None) in - # _mock_obtain_certificate(), which breaks this test. - # Thus fixed by the next line. - + def _authzr_from_domains(self, domains): authzr = [] # domain ordering should not be affected by authorization order - for domain in reversed(self.eg_domains): + for domain in reversed(domains): authzr.append( mock.MagicMock( body=mock.MagicMock( identifier=mock.MagicMock( value=domain)))) + return authzr + + def _test_obtain_certificate_common(self, key, csr, authzr_ret=None, auth_count=1): + self._mock_obtain_certificate() + + # return_value is essentially set to (None, None) in + # _mock_obtain_certificate(), which breaks this test. + # Thus fixed by the next line. + authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr @@ -281,7 +302,7 @@ class ClientTest(ClientTestCommon): self.assertEqual( result, (mock.sentinel.certr, mock.sentinel.chain, key, csr)) - self._check_obtain_certificate() + self._check_obtain_certificate(auth_count) @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') From ea3b78e3c9ff11da1d6919f2c5c36d53623512d3 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 20 Feb 2018 18:53:48 -0800 Subject: [PATCH 84/98] update order object with returned authorizations (#5598) --- certbot/client.py | 6 +++--- certbot/tests/client_test.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 404e1e0d9..dd11f2204 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -261,9 +261,8 @@ class Client(object): if orderr is None: orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr) - else: - authzr = orderr.authorizations - + orderr = orderr.update(authorizations=authzr) + authzr = orderr.authorizations certr = self.acme.request_issuance( jose.ComparableX509( @@ -320,6 +319,7 @@ class Client(object): orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) + orderr = orderr.update(authorizations=authzr) auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f4a8a5c8a..a65341692 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -294,6 +294,7 @@ class ClientTest(ClientTestCommon): authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr + self.eg_order.update().authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): From f1b7017c0c632054c61a231f8cd83db092d063fa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Feb 2018 16:20:45 -0800 Subject: [PATCH 85/98] Finish dropping Python 2.6 and 3.3 support * Undo letsencrypt-auto changes * Remove ordereddict import * Add Python 3.4 tests to replace 3.3 * Add python_requires * update pipstrap --- .travis.yml | 4 + acme/setup.py | 1 + certbot-apache/setup.py | 1 + certbot-compatibility-test/setup.py | 1 + certbot-dns-cloudflare/setup.py | 1 + certbot-dns-cloudxns/setup.py | 1 + certbot-dns-digitalocean/setup.py | 1 + certbot-dns-dnsimple/setup.py | 1 + certbot-dns-dnsmadeeasy/setup.py | 1 + certbot-dns-google/setup.py | 1 + certbot-dns-luadns/setup.py | 1 + certbot-dns-nsone/setup.py | 1 + certbot-dns-rfc2136/setup.py | 1 + certbot-dns-route53/setup.py | 1 + certbot-nginx/setup.py | 1 + certbot/plugins/disco.py | 8 +- certbot/util.py | 8 +- letsencrypt-auto-source/letsencrypt-auto | 124 ++++++++++++------ .../letsencrypt-auto.template | 33 +++-- letsencrypt-auto-source/pieces/pipstrap.py | 91 ++++++++----- letshelp-certbot/setup.py | 1 + setup.py | 1 + 22 files changed, 192 insertions(+), 92 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1077d99d9..42b8d679d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,10 @@ matrix: env: TOXENV=py27-oldest sudo: required services: docker + - python: "3.4" + env: TOXENV=py34 + sudo: required + services: docker - python: "3.6" env: TOXENV=py36 sudo: required diff --git a/acme/setup.py b/acme/setup.py index ba5c8e6fb..1f16f3b99 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -45,6 +45,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index d7c223a0a..86b0c646e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 7e1b059e2..861921ef7 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -34,6 +34,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d619f1872..6db6cc48f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 5d14f3e29..bf337c3d0 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index ce8fedd46..12d55f660 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 06af16759..79c93c942 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 7c0f3ed86..5d0970af1 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index de881ad84..cdfa205aa 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -36,6 +36,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 0d580b7ee..6c0dfb68f 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index c0ba11470..09a4e2cf7 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 5161e7a94..06efc373d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 09f8a7d52..8bd157166 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 37c477ef6..58f687aea 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 37baf98f7..5a7e07ec0 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -5,6 +5,8 @@ import logging import pkg_resources import six +from collections import OrderedDict + import zope.interface import zope.interface.verify @@ -12,12 +14,6 @@ from certbot import constants from certbot import errors from certbot import interfaces -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) diff --git a/certbot/util.py b/certbot/util.py index b7e60a225..b81799373 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -16,18 +16,14 @@ import stat import subprocess import sys +from collections import OrderedDict + import configargparse from certbot import constants from certbot import errors from certbot import lock -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index aed15a8ef..85659cfad 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -781,11 +781,20 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -956,10 +965,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -1220,6 +1237,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -1253,33 +1271,32 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 0 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), +# Pip has no dependencies, as it vendors everything: +PIP_PACKAGE = [ + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] + + +OTHER_PACKAGES = maybe_argparse + [ # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -1300,12 +1317,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -1315,8 +1333,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -1329,6 +1348,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -1336,17 +1373,24 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # We download and install pip first, then the rest, to avoid the bug + # https://github.com/certbot/certbot/issues/4938. + pip_downloads, other_downloads = [ + [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in packages] + for packages in (PIP_PACKAGE, OTHER_PACKAGES)] + for downloads in (pip_downloads, other_downloads): + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it + # otherwise sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index b3d6ab740..618e8f6bd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -320,11 +320,20 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -495,10 +504,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 78491b5e3..ed55b37e9 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -23,6 +23,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -56,33 +57,32 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 0 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), +# Pip has no dependencies, as it vendors everything: +PIP_PACKAGE = [ + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] + + +OTHER_PACKAGES = maybe_argparse + [ # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -103,12 +103,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -118,8 +119,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -132,6 +134,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -139,17 +159,24 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # We download and install pip first, then the rest, to avoid the bug + # https://github.com/certbot/certbot/issues/4938. + pip_downloads, other_downloads = [ + [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in packages] + for packages in (PIP_PACKAGE, OTHER_PACKAGES)] + for downloads in (pip_downloads, other_downloads): + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it + # otherwise sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 7c8c39068..b5be07a59 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: System Administrators', diff --git a/setup.py b/setup.py index e3824a7f7..f314449e6 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Console', From f3b23662f1fddedcea702c6ea5b54d5f97d1b0a8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 21 Feb 2018 20:52:04 -0800 Subject: [PATCH 86/98] Don't error immediately on wildcards. (#5600) --- certbot/tests/display/ops_test.py | 4 ++-- certbot/tests/main_test.py | 7 ++----- certbot/util.py | 10 ---------- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 57d82f839..c4f58ba7c 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -300,8 +300,8 @@ class ChooseNamesTest(unittest.TestCase): from certbot.display.ops import get_valid_domains all_valid = ["example.com", "second.example.com", "also.example.com", "under_score.example.com", - "justtld"] - all_invalid = ["öóòps.net", "*.wildcard.com", "uniçodé.com"] + "justtld", "*.wildcard.com"] + all_invalid = ["öóòps.net", "uniçodé.com"] two_valid = ["example.com", "úniçøde.com", "also.example.com"] self.assertEqual(get_valid_domains(all_valid), all_valid) self.assertEqual(get_valid_domains(all_invalid), []) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 518653a53..c31a3fb33 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,3 +1,4 @@ +# coding=utf-8 """Tests for certbot.main.""" # pylint: disable=too-many-lines from __future__ import print_function @@ -939,10 +940,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertRaises(errors.ConfigurationError, self._call, ['-d', (('a' * 50) + '.') * 10]) - # Wildcard - self.assertRaises(errors.ConfigurationError, - self._call, - ['-d', '*.wildcard.tld']) # Bare IP address (this is actually a different error message now) self.assertRaises(errors.ConfigurationError, @@ -1232,7 +1229,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def test_renew_with_bad_domain(self): renewalparams = {'authenticator': 'webroot'} - names = ['*.example.com'] + names = ['uniçodé.com'] self._test_renew_common(renewalparams=renewalparams, error_expected=True, names=names, assert_oc_called=False) diff --git a/certbot/util.py b/certbot/util.py index b7e60a225..70f402a72 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -552,16 +552,6 @@ def enforce_domain_sanity(domain): :returns: The domain cast to `str`, with ASCII-only contents :rtype: str """ - if isinstance(domain, six.text_type): - wildcard_marker = u"*." - else: - wildcard_marker = b"*." - - # Check if there's a wildcard domain - if domain.startswith(wildcard_marker): - raise errors.ConfigurationError( - "Wildcard domains are not supported: {0}".format(domain)) - # Unicode try: if isinstance(domain, six.binary_type): From c3659c300b0720d29939331c1c7c86586fd629ae Mon Sep 17 00:00:00 2001 From: Marcus LaFerrera Date: Thu, 22 Feb 2018 13:09:06 -0500 Subject: [PATCH 87/98] Return str rather than bytes (#5585) * Return str rather than bytes Project id is returned as bytes, which causes issues when constructing the google cloud API url, converting `b'PROJECT_ID'` to `b%27PROJECT_ID%27` causing the request to fail. * Ensure we handle both bytes and str types * project_id should be a str or bytes, not int --- certbot-dns-google/certbot_dns_google/dns_google.py | 5 ++++- certbot-dns-google/certbot_dns_google/dns_google_test.py | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index 37fd6b0de..cea754c06 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -224,4 +224,7 @@ class _GoogleClient(object): if r.status != 200: raise ValueError("Invalid status code: {0}".format(r)) - return content + if isinstance(content, bytes): + return content.decode() + else: + return content diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 85649fc7f..53f84dd6e 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -223,9 +223,13 @@ class GoogleClientTest(unittest.TestCase): response = DummyResponse() response.status = 200 - with mock.patch('httplib2.Http.request', return_value=(response, 1234)): + with mock.patch('httplib2.Http.request', return_value=(response, 'test-test-1')): project_id = _GoogleClient.get_project_id() - self.assertEqual(project_id, 1234) + self.assertEqual(project_id, 'test-test-1') + + with mock.patch('httplib2.Http.request', return_value=(response, b'test-test-1')): + project_id = _GoogleClient.get_project_id() + self.assertEqual(project_id, 'test-test-1') failed_response = DummyResponse() failed_response.status = 404 From 457269b0052f68b7e1b4a75414175a2e22f27ae6 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 22 Feb 2018 10:14:29 -0800 Subject: [PATCH 88/98] Add finalize_order to shim object, update Certbot to use it (#5601) * update order object with returned authorizations * major structure of finalize_order shim refactor * util methods and imports for finalize_order shim refactor * update certbot.tests.client_test.py * extraneous client_test imports * remove correct import * update renewal call * add test for acme.dump_pyopenssl_chain * Add test for certbot.crypto_util.cert_and_chain_from_fullchain * add tests for acme.client and change to fetch chain failure to TimeoutError * s/rytpe/rtype * remove ClientV1 passthrough * dump the wrapped cert * remove dead code * remove the correct dead code * support earlier mock --- acme/acme/client.py | 43 ++++++++++++++-- acme/acme/client_test.py | 71 ++++++++++++++++++++++++- acme/acme/crypto_util.py | 22 ++++++++ acme/acme/crypto_util_test.py | 28 ++++++++++ certbot/client.py | 81 +++++++++-------------------- certbot/crypto_util.py | 25 +++++---- certbot/main.py | 4 +- certbot/renewal.py | 5 +- certbot/tests/client_test.py | 86 +++++++++++-------------------- certbot/tests/crypto_util_test.py | 13 +++++ certbot/tests/util.py | 5 -- 11 files changed, 240 insertions(+), 143 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 1838fab42..97f529aae 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -677,9 +677,6 @@ class BackwardsCompatibleClientV2(object): return getattr(self.client, name) elif name in dir(ClientBase): return getattr(self.client, name) - # temporary, for breaking changes into smaller pieces - elif name in dir(Client): - return getattr(self.client, name) else: raise AttributeError() @@ -723,10 +720,48 @@ class BackwardsCompatibleClientV2(object): authorizations = [] for domain in dnsNames: authorizations.append(self.client.request_domain_challenges(domain)) - return messages.OrderResource(authorizations=authorizations) + return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem) else: return self.client.new_order(csr_pem) + def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ + if self.acme_version == 1: + csr_pem = orderr.csr_pem + certr = self.client.request_issuance( + jose.ComparableX509( + OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)), + orderr.authorizations) + + chain = None + while datetime.datetime.now() < deadline: + try: + chain = self.client.fetch_chain(certr) + break + except errors.Error: + time.sleep(1) + + if chain is None: + raise errors.TimeoutError( + 'Failed to fetch chain. You should not deploy the generated ' + 'certificate, please rerun the command for a new one.') + + cert = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) + chain = crypto_util.dump_pyopenssl_chain(chain) + + return orderr.update(fullchain_pem=(cert + chain)) + else: + return self.client.finalize_order(orderr, deadline) + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1b33ca5d7..acc5193ca 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -8,6 +8,7 @@ from six.moves import http_client # pylint: disable=import-error import josepy as jose import mock +import OpenSSL import requests from acme import challenges @@ -82,6 +83,29 @@ class ClientTestBase(unittest.TestCase): class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" + def setUp(self): + super(BackwardsCompatibleClientV2Test, self).setUp() + # contains a loaded cert + self.certr = messages.CertificateResource( + body=messages_test.CERT) + + loaded = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, CERT_SAN_PEM) + wrapped = jose.ComparableX509(loaded) + self.chain = [wrapped, wrapped] + + self.cert_pem = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped) + + single_chain = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, loaded) + self.chain_pem = single_chain + single_chain + + self.fullchain_pem = self.cert_pem + self.chain_pem + + self.orderr = messages.OrderResource( + csr_pem=CSR_SAN_PEM) + def _init(self): uri = 'http://www.letsencrypt-demo.org/directory' from acme.client import BackwardsCompatibleClientV2 @@ -109,8 +133,6 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client = self._init() self.assertEqual(client.directory, client.client.directory) self.assertEqual(client.key, KEY) - # delete this line once we finish migrating to new API: - self.assertEqual(client.register, client.client.register) self.assertEqual(client.update_registration, client.client.update_registration) self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') @@ -182,7 +204,52 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.new_order(mock_csr_pem) mock_client().new_order.assert_called_once_with(mock_csr_pem) + @mock.patch('acme.client.Client') + def test_finalize_order_v1_success(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + mock_client().fetch_chain.assert_called_once_with(self.certr) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_fetch_chain_error(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + mock_client().fetch_chain.side_effect = [errors.Error, self.chain] + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + self.assertEqual(mock_client().fetch_chain.call_count, 2) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_timeout(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + + deadline = deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + client = self._init() + self.assertRaises(errors.TimeoutError, client.finalize_order, + self.orderr, deadline) + + def test_finalize_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_orderr = mock.MagicMock() + mock_deadline = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.finalize_order(mock_orderr, mock_deadline) + mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline) class ClientTest(ClientTestBase): diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index a986721f0..f13c5109c 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -8,6 +8,8 @@ import socket import sys import OpenSSL +import josepy as jose + from acme import errors @@ -280,3 +282,23 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_pubkey(key) cert.sign(key, "sha256") return cert + +def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): + """Dump certificate chain into a bundle. + + :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in + :class:`josepy.util.ComparableX509`). + + """ + # XXX: returns empty string when no chain is available, which + # shuts up RenewableCert, but might not be the best solution... + + def _dump_cert(cert): + if isinstance(cert, jose.ComparableX509): + # pylint: disable=protected-access + cert = cert.wrapped + return OpenSSL.crypto.dump_certificate(filetype, cert) + + # assumes that OpenSSL.crypto.dump_certificate includes ending + # newline character + return b"".join(_dump_cert(cert) for cert in chain) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 14aaac8b5..e8dd3b20c 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -225,5 +225,33 @@ class MakeCSRTest(unittest.TestCase): self.assertEqual(len(must_staple_exts), 1, "Expected exactly one Must Staple extension") + +class DumpPyopensslChainTest(unittest.TestCase): + """Test for dump_pyopenssl_chain.""" + + @classmethod + def _call(cls, loaded): + # pylint: disable=protected-access + from acme.crypto_util import dump_pyopenssl_chain + return dump_pyopenssl_chain(loaded) + + def test_dump_pyopenssl_chain(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + length = sum( + len(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) + for cert in loaded) + self.assertEqual(len(self._call(loaded)), length) + + def test_dump_pyopenssl_chain_wrapped(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + wrap_func = jose.ComparableX509 + wrapped = [wrap_func(cert) for cert in loaded] + dump_func = OpenSSL.crypto.dump_certificate + length = sum(len(dump_func(OpenSSL.crypto.FILETYPE_PEM, cert)) for cert in loaded) + self.assertEqual(len(self._call(wrapped)), length) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/client.py b/certbot/client.py index dd11f2204..fc3848a5c 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -1,4 +1,5 @@ """Certbot client API.""" +import datetime import logging import os import platform @@ -11,7 +12,6 @@ import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util -from acme import errors as acme_errors from acme import messages import certbot @@ -243,8 +243,7 @@ class Client(object): than `authkey`. :param acme.messages.OrderResource orderr: contains authzrs - :returns: `.CertificateResource` and certificate chain (as - returned by `.fetch_chain`). + :returns: certificate and chain as PEM strings :rtype: tuple """ @@ -264,32 +263,9 @@ class Client(object): orderr = orderr.update(authorizations=authzr) authzr = orderr.authorizations - certr = self.acme.request_issuance( - jose.ComparableX509( - OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr.data)), - authzr) - - notify = zope.component.getUtility(interfaces.IDisplay).notification - retries = 0 - chain = None - - while retries <= 1: - if retries: - notify('Failed to fetch chain, please check your network ' - 'and continue', pause=True) - try: - chain = self.acme.fetch_chain(certr) - break - except acme_errors.Error: - logger.debug('Failed to fetch chain', exc_info=True) - retries += 1 - - if chain is None: - raise acme_errors.Error( - 'Failed to fetch chain. You should not deploy the generated ' - 'certificate, please rerun the command for a new one.') - - return certr, chain + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.acme.finalize_order(orderr, deadline) + return crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. @@ -298,10 +274,9 @@ class Client(object): :param list domains: domains to get a certificate - :returns: `.CertificateResource`, certificate chain (as - returned by `.fetch_chain`), and newly generated private key - (`.util.Key`) and DER-encoded Certificate Signing Request - (`.util.CSR`). + :returns: :returns: certificate as PEM string, chain as PEM string, + newly generated private key (`.util.Key`), and DER-encoded + Certificate Signing Request (`.util.CSR`). :rtype: tuple """ @@ -329,9 +304,9 @@ class Client(object): os.remove(csr.file) return self.obtain_certificate(successful_domains) else: - certr, chain = self.obtain_certificate_from_csr(csr, orderr) + cert, chain = self.obtain_certificate_from_csr(csr, orderr) - return certr, chain, key, csr + return cert, chain, key, csr # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): @@ -350,7 +325,7 @@ class Client(object): be obtained, or None if doing a successful dry run. """ - certr, chain, key, _ = self.obtain_certificate(domains) + cert, chain, key, _ = self.obtain_certificate(domains) if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): @@ -365,19 +340,16 @@ class Client(object): return None else: return storage.RenewableCert.new_lineage( - new_name, OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), - key.pem, crypto_util.dump_pyopenssl_chain(chain), + new_name, cert, + key.pem, chain, self.config) - def save_certificate(self, certr, chain_cert, + def save_certificate(self, cert_pem, chain_pem, cert_path, chain_path, fullchain_path): """Saves the certificate received from the ACME server. - :param certr: ACME "certificate" resource. - :type certr: :class:`acme.messages.Certificate` - - :param list chain_cert: + :param str cert_pem: + :param str chain_pem: :param str cert_path: Candidate path to a certificate. :param str chain_path: Candidate path to a certificate chain. :param str fullchain_path: Candidate path to a full cert chain. @@ -394,8 +366,6 @@ class Client(object): os.path.dirname(path), 0o755, os.geteuid(), self.config.strict_permissions) - cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) cert_file, abs_cert_path = _open_pem_file('cert_path', cert_path) @@ -406,20 +376,15 @@ class Client(object): logger.info("Server issued certificate; certificate written to %s", abs_cert_path) - if not chain_cert: - return abs_cert_path, None, None - else: - chain_pem = crypto_util.dump_pyopenssl_chain(chain_cert) + chain_file, abs_chain_path =\ + _open_pem_file('chain_path', chain_path) + fullchain_file, abs_fullchain_path =\ + _open_pem_file('fullchain_path', fullchain_path) - chain_file, abs_chain_path =\ - _open_pem_file('chain_path', chain_path) - fullchain_file, abs_fullchain_path =\ - _open_pem_file('fullchain_path', fullchain_path) + _save_chain(chain_pem, chain_file) + _save_chain(cert_pem + chain_pem, fullchain_file) - _save_chain(chain_pem, chain_file) - _save_chain(cert_pem + chain_pem, fullchain_file) - - return abs_cert_path, abs_chain_path, abs_fullchain_path + return abs_cert_path, abs_chain_path, abs_fullchain_path def deploy_certificate(self, domains, privkey_path, cert_path, chain_path, fullchain_path): diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 8368855cd..11721cc10 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,7 +14,6 @@ import six import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 -import josepy as jose from acme import crypto_util as acme_crypto_util @@ -367,16 +366,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """ # XXX: returns empty string when no chain is available, which # shuts up RenewableCert, but might not be the best solution... - - def _dump_cert(cert): - if isinstance(cert, jose.ComparableX509): - # pylint: disable=protected-access - cert = cert.wrapped - return OpenSSL.crypto.dump_certificate(filetype, cert) - - # assumes that OpenSSL.crypto.dump_certificate includes ending - # newline character - return b"".join(_dump_cert(cert) for cert in chain) + return acme_crypto_util.dump_pyopenssl_chain(chain, filetype) def notBefore(cert_path): @@ -443,3 +433,16 @@ def sha256sum(filename): with open(filename, 'rb') as f: sha256.update(f.read()) return sha256.hexdigest() + +def cert_and_chain_from_fullchain(fullchain_pem): + """Split fullchain_pem into cert_pem and chain_pem + + :param str fullchain_pem: concatenated cert + chain + + :returns: tuple of string cert_pem and chain_pem + :rtype: tuple + """ + cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, + OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + chain = fullchain_pem[len(cert):] + return (cert, chain) diff --git a/certbot/main.py b/certbot/main.py index d01f68920..eff4c9c8f 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1064,13 +1064,13 @@ def _csr_get_and_save_cert(config, le_client): """ csr, _ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(csr) + cert, chain = le_client.obtain_certificate_from_csr(csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) return None, None cert_path, _, fullchain_path = le_client.save_certificate( - certr, chain, config.cert_path, config.chain_path, config.fullchain_path) + cert, chain, config.cert_path, config.chain_path, config.fullchain_path) return cert_path, fullchain_path def renew_cert(config, plugins, lineage): diff --git a/certbot/renewal.py b/certbot/renewal.py index 024a815cc..ea5d87a5e 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -294,15 +294,12 @@ def renew_cert(config, domains, le_client, lineage): _avoid_invalidating_lineage(config, lineage, original_server) if not domains: domains = lineage.names() - new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) + new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert)) else: prior_version = lineage.latest_common_version() - new_cert = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped) - new_chain = crypto_util.dump_pyopenssl_chain(new_chain) # TODO: Check return value of save_successor lineage.save_successor(prior_version, new_cert, new_key.pem, new_chain, config) lineage.update_all_links_to(lineage.latest_common_version()) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a65341692..ed9c140e7 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,12 +4,8 @@ import shutil import tempfile import unittest -import josepy as jose -import OpenSSL import mock -from acme import errors as acme_errors - from certbot import account from certbot import errors from certbot import util @@ -134,7 +130,10 @@ class ClientTest(ClientTestCommon): self.config.allow_subset_of_names = False self.config.dry_run = False self.eg_domains = ["example.com", "www.example.com"] - self.eg_order = mock.MagicMock(authorizations=[None]) + self.eg_order = mock.MagicMock( + authorizations=[None], + fullchain_pem=mock.sentinel.fullchain_pem, + csr_pem=mock.sentinel.csr_pem) def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] @@ -143,9 +142,9 @@ class ClientTest(ClientTestCommon): def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() self.client.auth_handler.handle_authorizations.return_value = [None] - self.acme.request_issuance.return_value = mock.sentinel.certr - self.acme.fetch_chain.return_value = mock.sentinel.chain + self.acme.finalize_order.return_value = self.eg_order self.acme.new_order.return_value = self.eg_order + self.eg_order.update.return_value = self.eg_order def _check_obtain_certificate(self, auth_count=1): if auth_count == 1: @@ -155,27 +154,24 @@ class ClientTest(ClientTestCommon): else: self.assertEqual(self.client.auth_handler.handle_authorizations.call_count, auth_count) - authzr = self.client.auth_handler.handle_authorizations() - - self.acme.request_issuance.assert_called_once_with( - jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, CSR_SAN)), - authzr) - - self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) + self.acme.finalize_order.assert_called_once_with( + self.eg_order, mock.ANY) + @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.logger") @test_util.patch_get_utility() def test_obtain_certificate_from_csr(self, unused_mock_get_utility, - mock_logger): + mock_logger, mock_crypto_util): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) orderr = self.acme.new_order(test_csr.data) auth_handler.handle_authorizations(orderr, False) self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, orderr=orderr)) @@ -184,7 +180,7 @@ class ClientTest(ClientTestCommon): # Test for orderr=None self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, orderr=None)) @@ -198,41 +194,13 @@ class ClientTest(ClientTestCommon): test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_succeeded( - self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = [acme_errors.Error, - mock.sentinel.chain] - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - - orderr = self.acme.new_order(test_csr.data) - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - test_csr, - orderr=orderr)) - self.assertEqual(1, mock_get_utility().notification.call_count) - - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_failed(self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = acme_errors.Error - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - - orderr = self.acme.new_order(test_csr.data) - self.assertRaises( - acme_errors.Error, - self.client.obtain_certificate_from_csr, - test_csr, - orderr=orderr) - self.assertEqual(1, mock_get_utility().notification.call_count) - @mock.patch("certbot.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): csr = util.CSR(form="pem", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) self._test_obtain_certificate_common(mock.sentinel.key, csr) @@ -240,6 +208,8 @@ class ClientTest(ClientTestCommon): self.config.rsa_key_size, self.config.key_dir) mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) + mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with( + mock.sentinel.fullchain_pem) @mock.patch("certbot.client.crypto_util") @mock.patch("os.remove") @@ -248,6 +218,8 @@ class ClientTest(ClientTestCommon): key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = key + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) authzr = self._authzr_from_domains(["example.com"]) self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) @@ -255,6 +227,7 @@ class ClientTest(ClientTestCommon): self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) self.assertEqual(mock_crypto_util.init_save_csr.call_count, 2) self.assertEqual(mock_remove.call_count, 2) + self.assertEqual(mock_crypto_util.cert_and_chain_from_fullchain.call_count, 1) @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.acme_crypto_util") @@ -263,6 +236,8 @@ class ClientTest(ClientTestCommon): mock_acme_crypto.make_csr.return_value = CSR_SAN mock_crypto.make_key.return_value = mock.sentinel.key_pem key = util.Key(file=None, pem=mock.sentinel.key_pem) + mock_crypto.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) self.client.config.dry_run = True self._test_obtain_certificate_common(key, csr) @@ -272,6 +247,7 @@ class ClientTest(ClientTestCommon): mock.sentinel.key_pem, self.eg_domains, self.config.must_staple) mock_crypto.init_save_key.assert_not_called() mock_crypto.init_save_csr.assert_not_called() + self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1) def _authzr_from_domains(self, domains): authzr = [] @@ -294,7 +270,6 @@ class ClientTest(ClientTestCommon): authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr - self.eg_order.update().authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): @@ -302,13 +277,12 @@ class ClientTest(ClientTestCommon): self.assertEqual( result, - (mock.sentinel.certr, mock.sentinel.chain, key, csr)) + (mock.sentinel.cert, mock.sentinel.chain, key, csr)) self._check_obtain_certificate(auth_count) @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') - @mock.patch('OpenSSL.crypto.dump_certificate') - def test_obtain_and_enroll_certificate(self, mock_dump_certificate, + def test_obtain_and_enroll_certificate(self, mock_storage, mock_obtain_certificate): domains = ["example.com", "www.example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), @@ -324,7 +298,6 @@ class ClientTest(ClientTestCommon): self.assertFalse(self.client.obtain_and_enroll_certificate(domains, None)) self.assertTrue(mock_storage.call_count == 2) - self.assertTrue(mock_dump_certificate.call_count == 2) @mock.patch("certbot.cli.helpful_parser") def test_save_certificate(self, mock_parser): @@ -333,9 +306,8 @@ class ClientTest(ClientTestCommon): tmp_path = tempfile.mkdtemp() os.chmod(tmp_path, 0o755) # TODO: really?? - certr = mock.MagicMock(body=test_util.load_comparable_cert(certs[0])) - chain_cert = [test_util.load_comparable_cert(certs[0]), - test_util.load_comparable_cert(certs[1])] + cert_pem = test_util.load_vector(certs[0]) + chain_pem = (test_util.load_vector(certs[0]) + test_util.load_vector(certs[1])) candidate_cert_path = os.path.join(tmp_path, "certs", "cert_512.pem") candidate_chain_path = os.path.join(tmp_path, "chains", "chain.pem") candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") @@ -345,7 +317,7 @@ class ClientTest(ClientTestCommon): "--fullchain-path", candidate_fullchain_path] cert_path, chain_path, fullchain_path = self.client.save_certificate( - certr, chain_cert, candidate_cert_path, candidate_chain_path, + cert_pem, chain_pem, candidate_cert_path, candidate_chain_path, candidate_fullchain_path) self.assertEqual(os.path.dirname(cert_path), diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index f0e2c017e..00303fab3 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -373,5 +373,18 @@ class Sha256sumTest(unittest.TestCase): '914ffed8daf9e2c99d90ac95c77d54f32cbd556672facac380f0c063498df84e') +class CertAndChainFromFullchainTest(unittest.TestCase): + """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" + + def test_cert_and_chain_from_fullchain(self): + cert_pem = CERT + chain_pem = CERT + SS_CERT + fullchain_pem = cert_pem + chain_pem + from certbot.crypto_util import cert_and_chain_from_fullchain + cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem) + self.assertEqual(cert_out, cert_pem) + self.assertEqual(chain_out, chain_pem) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 60d8d6084..8434d11de 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -57,11 +57,6 @@ def load_cert(*names): return OpenSSL.crypto.load_certificate(loader, load_vector(*names)) -def load_comparable_cert(*names): - """Load ComparableX509 cert.""" - return jose.ComparableX509(load_cert(*names)) - - def load_csr(*names): """Load certificate request.""" loader = _guess_loader( From 990b211a76efb5a376cc4076bbbc2c2f0a2b3f2b Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 22 Feb 2018 12:33:55 -0800 Subject: [PATCH 89/98] Remove extra `:returns:` (#5611) --- certbot/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/client.py b/certbot/client.py index fc3848a5c..2d7288ce3 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -274,7 +274,7 @@ class Client(object): :param list domains: domains to get a certificate - :returns: :returns: certificate as PEM string, chain as PEM string, + :returns: certificate as PEM string, chain as PEM string, newly generated private key (`.util.Key`), and DER-encoded Certificate Signing Request (`.util.CSR`). :rtype: tuple From 1e46d26ac3511ca92e14c98c823a82286142ebb6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 22 Feb 2018 16:28:50 -0800 Subject: [PATCH 90/98] Fix ACMEv2 issues (#5612) * Add post wrapper to automatically add acme_version * Add uri to authzr. * Only add kid when account is set. * Add content_type when downloading certificate. * Only save new_authz URL when it exists. * Handle combinations in ACMEv1 and ACMEv2. * Add tests for ACMEv2 "combinations". --- acme/acme/client.py | 61 ++++++++++++-------- certbot/account.py | 17 ++++-- certbot/auth_handler.py | 12 +++- certbot/client.py | 4 +- certbot/tests/auth_handler_test.py | 93 ++++++++++++++++++++++++------ 5 files changed, 134 insertions(+), 53 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 97f529aae..9854aae31 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -70,7 +70,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes terms_of_service=terms_of_service) def _send_recv_regr(self, regr, body): - response = self.net.post(regr.uri, body, acme_version=self.acme_version) + response = self._post(regr.uri, body) # TODO: Boulder returns httplib.ACCEPTED #assert response.status_code == httplib.OK @@ -82,6 +82,13 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response, uri=regr.uri, terms_of_service=regr.terms_of_service) + def _post(self, *args, **kwargs): + """Wrapper around self.net.post that adds the acme_version. + + """ + kwargs.setdefault('acme_version', self.acme_version) + return self.net.post(*args, **kwargs) + def update_registration(self, regr, update=None): """Update registration. @@ -143,8 +150,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self.net.post(challb.uri, response, - acme_version=self.acme_version) + response = self._post(challb.uri, response) try: authzr_uri = response.links['up']['url'] except KeyError: @@ -216,12 +222,11 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .ClientError: If revocation is unsuccessful. """ - response = self.net.post(self.directory[messages.Revocation], + response = self._post(self.directory[messages.Revocation], messages.Revocation( certificate=cert, reason=rsn), - content_type=None, - acme_version=self.acme_version) + content_type=None) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') @@ -271,8 +276,7 @@ class Client(ClientBase): """ new_reg = messages.NewRegistration() if new_reg is None else new_reg - response = self.net.post(self.directory[new_reg], new_reg, - acme_version=1) + response = self._post(self.directory[new_reg], new_reg) # TODO: handle errors assert response.status_code == http_client.CREATED @@ -308,8 +312,7 @@ class Client(ClientBase): if new_authzr_uri is not None: logger.debug("request_challenges with new_authzr_uri deprecated.") new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz, new_authz, - acme_version=1) + response = self._post(self.directory.new_authz, new_authz) # TODO: handle errors assert response.status_code == http_client.CREATED return self._authzr_from_response(response, identifier) @@ -351,12 +354,11 @@ class Client(ClientBase): req = messages.CertificateRequest(csr=csr) content_type = DER_CONTENT_TYPE # TODO: add 'cert_type 'argument - response = self.net.post( + response = self._post( self.directory.new_cert, req, content_type=content_type, - headers={'Accept': content_type}, - acme_version=1) + headers={'Accept': content_type}) cert_chain_uri = response.links.get('up', {}).get('url') @@ -552,8 +554,7 @@ class ClientV2(ClientBase): :returns: Registration Resource. :rtype: `.RegistrationResource` """ - response = self.net.post(self.directory['newAccount'], new_account, - acme_version=2) + response = self._post(self.directory['newAccount'], new_account) # "Instance of 'Field' has no key/contact member" bug: # pylint: disable=no-member regr = self._regr_from_response(response) @@ -577,11 +578,11 @@ class ClientV2(ClientBase): identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=name)) order = messages.NewOrder(identifiers=identifiers) - response = self.net.post(self.directory['newOrder'], order) + response = self._post(self.directory['newOrder'], order) body = messages.Order.from_json(response.json()) authorizations = [] for url in body.authorizations: - authorizations.append(self._authzr_from_response(self.net.get(url))) + authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) return messages.OrderResource( body=body, uri=response.headers.get('Location'), @@ -643,7 +644,7 @@ class ClientV2(ClientBase): csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) - self.net.post(orderr.body.finalize, wrapped_csr) + self._post(orderr.body.finalize, wrapped_csr) while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) @@ -651,17 +652,29 @@ class ClientV2(ClientBase): if body.error is not None: raise errors.IssuanceError(body.error) if body.certificate is not None: - certificate_response = self.net.get(body.certificate).text + certificate_response = self.net.get(body.certificate, + content_type=DER_CONTENT_TYPE).text return orderr.update(body=body, fullchain_pem=certificate_response) raise errors.TimeoutError() class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but - supports V1 servers. + supports V1 servers. - :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint - :ivar .ClientBase client: either Client or ClientV2 + .. note:: While this class handles the majority of the differences + between versions of the ACME protocol, if you need to support an + ACME server based on version 3 or older of the IETF ACME draft + that uses combinations in authorizations (or lack thereof) to + signal that the client needs to complete something other than + any single challenge in the authorization to make it valid, the + user of this class needs to understand and handle these + differences themselves. This does not apply to either of Let's + Encrypt's endpoints where successfully completing any challenge + in an authorization will make it valid. + + :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint + :ivar .ClientBase client: either Client or ClientV2 """ def __init__(self, net, key, server): @@ -829,7 +842,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes } if acme_version == 2: kwargs["url"] = url - kwargs["kid"] = self.account["uri"] + # newAccount and revokeCert work without the kid + if self.account is not None: + kwargs["kid"] = self.account["uri"] kwargs["key"] = self.key # pylint: disable=star-args return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2) diff --git a/certbot/account.py b/certbot/account.py index 41e980097..70d9a7fc3 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -223,12 +223,17 @@ class AccountFileStorage(interfaces.AccountStorage): try: with open(self._regr_path(account_dir_path), "w") as regr_file: regr = account.regr - with_uri = RegistrationResourceWithNewAuthzrURI( - new_authzr_uri=acme.directory.new_authz, - body=regr.body, - uri=regr.uri, - terms_of_service=regr.terms_of_service) - regr_file.write(with_uri.json_dumps()) + # If we have a value for new-authz, save it for forwards + # compatibility with older versions of Certbot. If we don't + # have a value for new-authz, this is an ACMEv2 directory where + # an older version of Certbot won't work anyway. + if hasattr(acme.directory, "new-authz"): + regr = RegistrationResourceWithNewAuthzrURI( + new_authzr_uri=acme.directory.new_authz, + body=regr.body, + uri=regr.uri, + terms_of_service=regr.terms_of_service) + regr_file.write(regr.json_dumps()) if not regr_only: with util.safe_open(self._key_path(account_dir_path), "w", chmod=0o400) as key_file: diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 47d806b94..9cc10d4b4 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -24,7 +24,7 @@ class AuthHandler(object): :class:`~acme.challenges.Challenge` types :type auth: :class:`certbot.interfaces.IAuthenticator` - :ivar acme.client.Client acme: ACME client API. + :ivar acme.client.BackwardsCompatibleClientV2 acme: ACME client API. :ivar account: Client's Account :type account: :class:`certbot.account.Account` @@ -100,10 +100,16 @@ class AuthHandler(object): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") for dom in domains: + dom_challenges = self.authzr[dom].body.challenges + if self.acme.acme_version == 1: + combinations = self.authzr[dom].body.combinations + else: + combinations = tuple((i,) for i in range(len(dom_challenges))) + path = gen_challenge_path( - self.authzr[dom].body.challenges, + dom_challenges, self._get_chall_pref(dom), - self.authzr[dom].body.combinations) + combinations) dom_achalls = self._challenge_factory( dom, path) diff --git a/certbot/client.py b/certbot/client.py index 2d7288ce3..0f4fa760d 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -212,8 +212,8 @@ class Client(object): :ivar .IAuthenticator auth: Prepared (`.IAuthenticator.prepare`) authenticator that can solve ACME challenges. :ivar .IInstaller installer: Installer. - :ivar acme.client.Client acme: Optional ACME client API handle. - You might already have one from `register`. + :ivar acme.client.BackwardsCompatibleClientV2 acme: Optional ACME + client API handle. You might already have one from `register`. """ diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 3633b673d..394002206 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -81,6 +81,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_account = mock.Mock(key=util.Key("file_path", "PEM")) self.mock_net = mock.MagicMock(spec=acme_client.Client) + self.mock_net.acme_version = 1 self.handler = AuthHandler( self.mock_auth, self.mock_net, self.mock_account, []) @@ -90,13 +91,13 @@ class HandleAuthorizationsTest(unittest.TestCase): def tearDown(self): logging.disable(logging.NOTSET) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1(self, mock_poll): - mock_poll.side_effect = self._validate_all - - authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + def _test_name1_tls_sni_01_1_common(self, combos): + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos) mock_order = mock.MagicMock(authorizations=[authzr]) - authzr = self.handler.handle_authorizations(mock_order) + + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) @@ -112,8 +113,15 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) + def test_name1_tls_sni_01_1_acme_1(self): + self._test_name1_tls_sni_01_1_common(combos=True) + + def test_name1_tls_sni_01_1_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name1_tls_sni_01_1_common(combos=False) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll): + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self, mock_poll): mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) @@ -138,17 +146,43 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name3_tls_sni_01_3(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) - + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self, mock_poll): + self.mock_net.acme_version = 2 mock_poll.side_effect = self._validate_all + self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) + self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) + + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) + + self.assertEqual(self.mock_net.answer_challenge.call_count, 1) + + self.assertEqual(mock_poll.call_count, 1) + chall_update = mock_poll.call_args[0][0] + self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(len(chall_update.values()), 1) + + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + cleaned_up_achalls = self.mock_auth.cleanup.call_args[0][0] + self.assertEqual(len(cleaned_up_achalls), 1) + self.assertEqual(cleaned_up_achalls[0].typ, "tls-sni-01") + + # Length of authorizations list + self.assertEqual(len(authzr), 1) + + def _test_name3_tls_sni_01_3_common(self, combos): + self.mock_net.request_domain_challenges.side_effect = functools.partial( + gen_dom_authzr, challs=acme_util.CHALLENGES, combos=combos) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - authzr = self.handler.handle_authorizations(mock_order) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -167,6 +201,13 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 3) + def test_name3_tls_sni_01_3_common_acme_1(self): + self._test_name3_tls_sni_01_3_common(combos=True) + + def test_name3_tls_sni_01_3_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name3_tls_sni_01_3_common(combos=False) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_debug_challenges(self, mock_poll): zope.component.provideUtility( @@ -194,30 +235,44 @@ class HandleAuthorizationsTest(unittest.TestCase): mock_order = mock.MagicMock(authorizations=[]) self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_preferred_challenge_choice(self, mock_poll): - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + def _test_preferred_challenge_choice_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - self.handler.handle_authorizations(mock_order) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") - def test_preferred_challenges_not_supported(self): - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + def test_preferred_challenge_choice_common_acme_1(self): + self._test_preferred_challenge_choice_common(combos=True) + + def test_preferred_challenge_choice_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenge_choice_common(combos=False) + + def _test_preferred_challenges_not_supported_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] mock_order = mock.MagicMock(authorizations=authzrs) self.handler.pref_challs.append(challenges.HTTP01.typ) self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def test_preferred_challenges_not_supported_acme_1(self): + self._test_preferred_challenges_not_supported_common(combos=True) + + def test_preferred_challenges_not_supported_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenges_not_supported_common(combos=False) + def _validate_all(self, unused_1, unused_2): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] From f3a0deba840f1c6bc1510ee71a0e70035fa8488c Mon Sep 17 00:00:00 2001 From: Nick Bebout Date: Fri, 23 Feb 2018 15:26:11 -0600 Subject: [PATCH 91/98] Remove min version of setuptools (#5617) --- acme/setup.py | 4 +--- certbot-apache/setup.py | 4 +--- certbot-dns-cloudflare/setup.py | 4 +--- certbot-dns-cloudxns/setup.py | 4 +--- certbot-dns-digitalocean/setup.py | 4 +--- certbot-dns-dnsimple/setup.py | 4 +--- certbot-dns-dnsmadeeasy/setup.py | 4 +--- certbot-dns-google/setup.py | 4 +--- certbot-dns-luadns/setup.py | 4 +--- certbot-dns-nsone/setup.py | 4 +--- certbot-dns-rfc2136/setup.py | 4 +--- certbot-dns-route53/setup.py | 4 +--- certbot-nginx/setup.py | 4 +--- setup.py | 4 +--- 14 files changed, 14 insertions(+), 42 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index ce426cf74..51bbf0f71 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -19,9 +19,7 @@ install_requires = [ 'pyrfc3339', 'pytz', 'requests[security]>=2.4.1', # security extras added in 2.4.1 - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible ] diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 38f41e9f1..76d7f5ca5 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'mock', 'python-augeas', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 612e7259f..e1b84d1be 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'cloudflare>=1.5.1', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3157400c6..53ceb58ea 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1a68400fa..3330bdd67 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'mock', 'python-digitalocean>=1.11', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six', 'zope.interface', ] diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 35de47308..00a3c032a 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a946d00a4..36119ade0 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8585fc848..10fccd1ea 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -15,9 +15,7 @@ install_requires = [ 'mock', # for oauth2client.service_account.ServiceAccountCredentials 'oauth2client>=2.0', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', # already a dependency of google-api-python-client, but added for consistency 'httplib2' diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4fec37e29..b094e1818 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index dca9ebf27..e777d821b 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bfa72b50b..6fc6dca73 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dnspython', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8df687972..0fbeab31b 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -10,9 +10,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'boto3', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 152f77de8..a84efe2c3 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -13,9 +13,7 @@ install_requires = [ 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/setup.py b/setup.py index 47b5b0b2c..736ef467f 100644 --- a/setup.py +++ b/setup.py @@ -46,9 +46,7 @@ install_requires = [ 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] From 6f86267a26c9b748bd90113fe62157a9a455cdd2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 12:42:13 -0800 Subject: [PATCH 92/98] Fix revocation in ACMEv2 (#5626) * Allow revoke to pass in a url * Add revocation support to ACMEv2. * Provide regr for account based revocation. * Add revoke wrapper to BackwardsCompat client --- acme/acme/client.py | 52 +++++++++++++++++++++++++++++++++++----- acme/acme/client_test.py | 26 +++++++++++++++++--- certbot/main.py | 4 ++-- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 9854aae31..e3f6e845d 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -211,7 +211,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response, authzr.body.identifier, authzr.uri) return updated_authzr, response - def revoke(self, cert, rsn): + def _revoke(self, cert, rsn, url): """Revoke certificate. :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in @@ -219,14 +219,16 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :param int rsn: Reason code for certificate revocation. + :param str url: ACME URL to post to + :raises .ClientError: If revocation is unsuccessful. """ - response = self._post(self.directory[messages.Revocation], - messages.Revocation( - certificate=cert, - reason=rsn), - content_type=None) + response = self._post(url, + messages.Revocation( + certificate=cert, + reason=rsn), + content_type=None) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') @@ -528,6 +530,18 @@ class Client(ClientBase): "Recursion limit reached. Didn't get {0}".format(uri)) return chain + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self._revoke(cert, rsn, self.directory[messages.Revocation]) class ClientV2(ClientBase): @@ -657,6 +671,19 @@ class ClientV2(ClientBase): return orderr.update(body=body, fullchain_pem=certificate_response) raise errors.TimeoutError() + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self._revoke(cert, rsn, self.directory['revokeCert']) + class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but @@ -775,6 +802,19 @@ class BackwardsCompatibleClientV2(object): else: return self.client.finalize_order(orderr, deadline) + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self.client.revoke(cert, rsn) + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index acc5193ca..1e4db2884 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -40,6 +40,7 @@ DIRECTORY_V2 = messages.Directory({ 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce', 'newOrder': 'https://www.letsencrypt-demo.org/acme/new-order', + 'revokeCert': 'https://www.letsencrypt-demo.org/acme/revoke-cert', }) @@ -79,6 +80,9 @@ class ClientTestBase(unittest.TestCase): self.authzr = messages.AuthorizationResource( body=self.authz, uri=authzr_uri) + # Reason code for revocation + self.rsn = 1 + class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" @@ -251,6 +255,19 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.finalize_order(mock_orderr, mock_deadline) mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline) + def test_revoke(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" @@ -271,9 +288,6 @@ class ClientTest(ClientTestBase): uri='https://www.letsencrypt-demo.org/acme/cert/1', cert_chain_uri='https://www.letsencrypt-demo.org/ca') - # Reason code for revocation - self.rsn = 1 - from acme.client import Client self.client = Client( directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) @@ -752,6 +766,12 @@ class ClientV2Test(ClientTestBase): deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) self.assertRaises(errors.TimeoutError, self.client.finalize_order, self.orderr, deadline) + def test_revoke(self): + self.client.revoke(messages_test.CERT, self.rsn) + self.net.post.assert_called_once_with( + self.directory["revokeCert"], mock.ANY, content_type=None, + acme_version=2) + class MockJSONDeSerializable(jose.JSONDeSerializable): # pylint: disable=missing-docstring diff --git a/certbot/main.py b/certbot/main.py index 33c7730c6..7be852e83 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -982,11 +982,11 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config config.cert_path[0], config.key_path[0]) crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) key = jose.JWK.load(config.key_path[1]) + acme = client.acme_from_config_key(config, key) else: # revocation by account key logger.debug("Revoking %s using Account Key", config.cert_path[0]) acc, _ = _determine_account(config) - key = acc.key - acme = client.acme_from_config_key(config, key) + acme = client.acme_from_config_key(config, acc.key, acc.regr) cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] logger.debug("Reason code for revocation: %s", config.reason) From b18696b6a0967ddb609aeb5841b93e6214099080 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 16:47:43 -0800 Subject: [PATCH 93/98] Don't run tests with Python 2.6 (#5627) * Don't run tests with Python 2.6. * Revert "Don't run tests with Python 2.6." This reverts commit 4a9d778cca62ae2bec4cf060726e88f1fd66f374. * Revert changes to auto_test.py. --- letsencrypt-auto-source/tests/auto_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 8c2bfc079..d187452a1 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -287,8 +287,8 @@ class AutoTests(TestCase): self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: - self.assertIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertTrue('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This # conveniently sets us up to test the next 2 cases. @@ -296,8 +296,8 @@ class AutoTests(TestCase): # Test when neither phase-1 upgrade nor phase-2 upgrade is # needed (probably a common case): out, err = run_letsencrypt_auto() - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertNotIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertFalse('Creating virtual environment...' in out) def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" @@ -312,8 +312,8 @@ class AutoTests(TestCase): # Create venv saving the correct bootstrap script version out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: bootstrap_version = f.read() @@ -329,8 +329,8 @@ class AutoTests(TestCase): out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) def test_openssl_failure(self): """Make sure we stop if the openssl signature check fails.""" From a39d2fe55b760707718dcd8225b7dc42dcef4c9c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 18:05:33 -0800 Subject: [PATCH 94/98] Fix wildcard issuance (#5620) * Add is_wildcard_domain to certbot.util. * Error with --allow-subset-of-names and wildcards. * Fix issue preventing wildcard cert issuance. * Kill assumption domain is unique in auth_handler * fix typo and add test * update comments --- certbot/auth_handler.py | 158 ++++++++++++++++------------- certbot/cli.py | 5 + certbot/client.py | 7 +- certbot/tests/auth_handler_test.py | 80 +++++++-------- certbot/tests/cli_test.py | 4 + certbot/tests/client_test.py | 1 + certbot/tests/util_test.py | 20 ++++ certbot/util.py | 18 ++++ 8 files changed, 180 insertions(+), 113 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 9cc10d4b4..2b38e4af5 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -1,4 +1,5 @@ """ACME AuthHandler.""" +import collections import logging import time @@ -17,6 +18,10 @@ from certbot import interfaces logger = logging.getLogger(__name__) +AnnotatedAuthzr = collections.namedtuple("AnnotatedAuthzr", ["authzr", "achalls"]) +"""Stores an authorization resource and its active annotated challenges.""" + + class AuthHandler(object): """ACME Authorization Handler for a client. @@ -29,10 +34,8 @@ class AuthHandler(object): :ivar account: Client's Account :type account: :class:`certbot.account.Account` - :ivar dict authzr: ACME Authorization Resource dict where keys are domains - and values are :class:`acme.messages.AuthorizationResource` - :ivar list achalls: DV challenges in the form of - :class:`certbot.achallenges.AnnotatedChallenge` + :ivar aauthzrs: ACME Authorization Resources and their active challenges + :type aauthzrs: `list` of `AnnotatedAuthzr` :ivar list pref_challs: sorted user specified preferred challenges type strings with the most preferred challenge listed first @@ -42,12 +45,9 @@ class AuthHandler(object): self.acme = acme self.account = account - self.authzr = dict() + self.aauthzrs = [] self.pref_challs = pref_challs - # List must be used to keep responses straight. - self.achalls = [] - def handle_authorizations(self, orderr, best_effort=False): """Retrieve all authorizations for challenges. @@ -63,17 +63,15 @@ class AuthHandler(object): authorizations """ - authzrs = orderr.authorizations - for authzr in authzrs: - self.authzr[authzr.body.identifier.value] = authzr - domains = self.authzr.keys() + for authzr in orderr.authorizations: + self.aauthzrs.append(AnnotatedAuthzr(authzr, [])) - self._choose_challenges(domains) + self._choose_challenges() config = zope.component.getUtility(interfaces.IConfig) notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... - while self.achalls: + while self._has_challenges(): resp = self._solve_challenges() logger.info("Waiting for verification...") if config.debug_challenges: @@ -87,8 +85,8 @@ class AuthHandler(object): self.verify_authzr_complete() # Only return valid authorizations - retVal = [authzr for authzr in self.authzr.values() - if authzr.body.status == messages.STATUS_VALID] + retVal = [aauthzr.authzr for aauthzr in self.aauthzrs + if aauthzr.authzr.body.status == messages.STATUS_VALID] if not retVal: raise errors.AuthorizationError( @@ -96,41 +94,54 @@ class AuthHandler(object): return retVal - def _choose_challenges(self, domains): + def _choose_challenges(self): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") - for dom in domains: - dom_challenges = self.authzr[dom].body.challenges + for aauthzr in self.aauthzrs: + aauthzr_challenges = aauthzr.authzr.body.challenges if self.acme.acme_version == 1: - combinations = self.authzr[dom].body.combinations + combinations = aauthzr.authzr.body.combinations else: - combinations = tuple((i,) for i in range(len(dom_challenges))) + combinations = tuple((i,) for i in range(len(aauthzr_challenges))) path = gen_challenge_path( - dom_challenges, - self._get_chall_pref(dom), + aauthzr_challenges, + self._get_chall_pref(aauthzr.authzr.body.identifier.value), combinations) - dom_achalls = self._challenge_factory( - dom, path) - self.achalls.extend(dom_achalls) + aauthzr_achalls = self._challenge_factory( + aauthzr.authzr, path) + aauthzr.achalls.extend(aauthzr_achalls) + + def _has_challenges(self): + """Do we have any challenges to perform?""" + return any(aauthzr.achalls for aauthzr in self.aauthzrs) def _solve_challenges(self): """Get Responses for challenges from authenticators.""" resp = [] + all_achalls = self._get_all_achalls() with error_handler.ErrorHandler(self._cleanup_challenges): try: - if self.achalls: - resp = self.auth.perform(self.achalls) + if all_achalls: + resp = self.auth.perform(all_achalls) except errors.AuthorizationError: logger.critical("Failure in setting up challenges.") logger.info("Attempting to clean up outstanding challenges...") raise - assert len(resp) == len(self.achalls) + assert len(resp) == len(all_achalls) return resp + def _get_all_achalls(self): + """Return all active challenges.""" + all_achalls = [] + for aauthzr in self.aauthzrs: + all_achalls.extend(aauthzr.achalls) + + return all_achalls + def _respond(self, resp, best_effort): """Send/Receive confirmation of all challenges. @@ -139,69 +150,67 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 chall_update = dict() - active_achalls = self._send_responses(self.achalls, - resp, chall_update) + active_achalls = self._send_responses(resp, chall_update) # Check for updated status... try: self._poll_challenges(chall_update, best_effort) finally: - # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) - def _send_responses(self, achalls, resps, chall_update): + def _send_responses(self, resps, chall_update): """Send responses and make sure errors are handled. :param dict chall_update: parameter that is updated to hold - authzr -> list of outstanding solved annotated challenges + aauthzr index to list of outstanding solved annotated challenges """ active_achalls = [] - for achall, resp in six.moves.zip(achalls, resps): - # This line needs to be outside of the if block below to - # ensure failed challenges are cleaned up correctly - active_achalls.append(achall) + resps_iter = iter(resps) + for i, aauthzr in enumerate(self.aauthzrs): + for achall in aauthzr.achalls: + # This line needs to be outside of the if block below to + # ensure failed challenges are cleaned up correctly + active_achalls.append(achall) - # Don't send challenges for None and False authenticator responses - if resp is not None and resp: - self.acme.answer_challenge(achall.challb, resp) - # TODO: answer_challenge returns challr, with URI, - # that can be used in _find_updated_challr - # comparisons... - if achall.domain in chall_update: - chall_update[achall.domain].append(achall) - else: - chall_update[achall.domain] = [achall] + resp = next(resps_iter) + # Don't send challenges for None and False authenticator responses + if resp: + self.acme.answer_challenge(achall.challb, resp) + # TODO: answer_challenge returns challr, with URI, + # that can be used in _find_updated_challr + # comparisons... + chall_update.setdefault(i, []).append(achall) return active_achalls def _poll_challenges( self, chall_update, best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" - dom_to_check = set(chall_update.keys()) - comp_domains = set() + indices_to_check = set(chall_update.keys()) + comp_indices = set() rounds = 0 - while dom_to_check and rounds < max_rounds: + while indices_to_check and rounds < max_rounds: # TODO: Use retry-after... time.sleep(min_sleep) all_failed_achalls = set() - for domain in dom_to_check: + for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( - domain, chall_update[domain]) + index, chall_update[index]) - if len(comp_achalls) == len(chall_update[domain]): - comp_domains.add(domain) + if len(comp_achalls) == len(chall_update[index]): + comp_indices.add(index) elif not failed_achalls: for achall, _ in comp_achalls: - chall_update[domain].remove(achall) + chall_update[index].remove(achall) # We failed some challenges... damage control else: if best_effort: - comp_domains.add(domain) + comp_indices.add(index) logger.warning( "Challenge failed for domain %s", - domain) + self.aauthzrs[index].authzr.body.identifier.value) else: all_failed_achalls.update( updated for _, updated in failed_achalls) @@ -210,24 +219,26 @@ class AuthHandler(object): _report_failed_challs(all_failed_achalls) raise errors.FailedChallenges(all_failed_achalls) - dom_to_check -= comp_domains - comp_domains.clear() + indices_to_check -= comp_indices + comp_indices.clear() rounds += 1 - def _handle_check(self, domain, achalls): + def _handle_check(self, index, achalls): """Returns tuple of ('completed', 'failed').""" completed = [] failed = [] - self.authzr[domain], _ = self.acme.poll(self.authzr[domain]) - if self.authzr[domain].body.status == messages.STATUS_VALID: + original_aauthzr = self.aauthzrs[index] + updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) + self.aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) + if updated_authzr.body.status == messages.STATUS_VALID: return achalls, [] # Note: if the whole authorization is invalid, the individual failed # challenges will be determined here... for achall in achalls: updated_achall = achall.update(challb=self._find_updated_challb( - self.authzr[domain], achall)) + updated_authzr, achall)) # This does nothing for challenges that have yet to be decided yet. if updated_achall.status == messages.STATUS_VALID: @@ -285,14 +296,17 @@ class AuthHandler(object): logger.info("Cleaning up challenges") if achall_list is None: - achalls = self.achalls + achalls = self._get_all_achalls() else: achalls = achall_list if achalls: self.auth.cleanup(achalls) for achall in achalls: - self.achalls.remove(achall) + for aauthzr in self.aauthzrs: + if achall in aauthzr.achalls: + aauthzr.achalls.remove(achall) + break def verify_authzr_complete(self): """Verifies that all authorizations have been decided. @@ -301,15 +315,16 @@ class AuthHandler(object): :rtype: bool """ - for authzr in self.authzr.values(): + for aauthzr in self.aauthzrs: + authzr = aauthzr.authzr if (authzr.body.status != messages.STATUS_VALID and authzr.body.status != messages.STATUS_INVALID): raise errors.AuthorizationError("Incomplete authorizations") - def _challenge_factory(self, domain, path): + def _challenge_factory(self, authzr, path): """Construct Namedtuple Challenges - :param str domain: domain of the enrollee + :param messages.AuthorizationResource authzr: authorization :param list path: List of indices from `challenges`. @@ -323,8 +338,9 @@ class AuthHandler(object): achalls = [] for index in path: - challb = self.authzr[domain].body.challenges[index] - achalls.append(challb_to_achall(challb, self.account.key, domain)) + challb = authzr.body.challenges[index] + achalls.append(challb_to_achall( + challb, self.account.key, authzr.body.identifier.value)) return achalls diff --git a/certbot/cli.py b/certbot/cli.py index 09dd71d13..1c2273c8a 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -599,6 +599,11 @@ class HelpfulArgumentParser(object): if parsed_args.validate_hooks: hooks.validate_hooks(parsed_args) + if parsed_args.allow_subset_of_names: + if any(util.is_wildcard_domain(d) for d in parsed_args.domains): + raise errors.Error("Using --allow-subset-of-names with a" + " wildcard domain is not supported.") + possible_deprecation_warning(parsed_args) return parsed_args diff --git a/certbot/client.py b/certbot/client.py index 0f4fa760d..81fc0b802 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -298,7 +298,12 @@ class Client(object): auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] - if successful_domains != domains: + # allow_subset_of_names is currently disabled for wildcard + # certificates. The reason for this and checking allow_subset_of_names + # below is because successful_domains == domains is never true if + # domains contains a wildcard because the ACME spec forbids identifiers + # in authzs from containing a wildcard character. + if self.config.allow_subset_of_names and successful_domains != domains: if not self.config.dry_run: os.remove(key.file) os.remove(csr.file) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 394002206..7650d2c95 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -29,32 +29,31 @@ class ChallengeFactoryTest(unittest.TestCase): # Account is mocked... self.handler = AuthHandler(None, None, mock.Mock(key="mock_key"), []) - self.dom = "test" - self.handler.authzr[self.dom] = acme_util.gen_authzr( - messages.STATUS_PENDING, self.dom, acme_util.CHALLENGES, + self.authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", acme_util.CHALLENGES, [messages.STATUS_PENDING] * 6, False) def test_all(self): achalls = self.handler._challenge_factory( - self.dom, range(0, len(acme_util.CHALLENGES))) + self.authzr, range(0, len(acme_util.CHALLENGES))) self.assertEqual( [achall.chall for achall in achalls], acme_util.CHALLENGES) def test_one_tls_sni(self): - achalls = self.handler._challenge_factory(self.dom, [1]) + achalls = self.handler._challenge_factory(self.authzr, [1]) self.assertEqual( [achall.chall for achall in achalls], [acme_util.TLSSNI01]) def test_unrecognized(self): - self.handler.authzr["failure.com"] = acme_util.gen_authzr( - messages.STATUS_PENDING, "failure.com", - [mock.Mock(chall="chall", typ="unrecognized")], - [messages.STATUS_PENDING]) + authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", + [mock.Mock(chall="chall", typ="unrecognized")], + [messages.STATUS_PENDING]) self.assertRaises( - errors.Error, self.handler._challenge_factory, "failure.com", [0]) + errors.Error, self.handler._challenge_factory, authzr, [0]) class HandleAuthorizationsTest(unittest.TestCase): @@ -103,7 +102,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -134,7 +133,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -160,7 +159,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -190,12 +189,12 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] self.assertEqual(len(list(six.iterkeys(chall_update))), 3) - self.assertTrue("0" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["0"]), 1) - self.assertTrue("1" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["1"]), 1) - self.assertTrue("2" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["2"]), 1) + self.assertTrue(0 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[0]), 1) + self.assertTrue(1 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[1]), 1) + self.assertTrue(2 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[2]), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -274,14 +273,15 @@ class HandleAuthorizationsTest(unittest.TestCase): self._test_preferred_challenges_not_supported_common(combos=False) def _validate_all(self, unused_1, unused_2): - for dom in six.iterkeys(self.handler.authzr): - azr = self.handler.authzr[dom] - self.handler.authzr[dom] = acme_util.gen_authzr( + for i, aauthzr in enumerate(self.handler.aauthzrs): + azr = aauthzr.authzr + updated_azr = acme_util.gen_authzr( messages.STATUS_VALID, - dom, + azr.body.identifier.value, [challb.chall for challb in azr.body.challenges], [messages.STATUS_VALID] * len(azr.body.challenges), azr.body.combinations) + self.handler.aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) class PollChallengesTest(unittest.TestCase): @@ -290,7 +290,7 @@ class PollChallengesTest(unittest.TestCase): def setUp(self): from certbot.auth_handler import challb_to_achall - from certbot.auth_handler import AuthHandler + from certbot.auth_handler import AuthHandler, AnnotatedAuthzr # Account and network are mocked... self.mock_net = mock.MagicMock() @@ -298,40 +298,38 @@ class PollChallengesTest(unittest.TestCase): None, self.mock_net, mock.Mock(key="mock_key"), []) self.doms = ["0", "1", "2"] - self.handler.authzr[self.doms[0]] = acme_util.gen_authzr( + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[0], [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False) - - self.handler.authzr[self.doms[1]] = acme_util.gen_authzr( + [messages.STATUS_PENDING] * 2, False), [])) + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) - - self.handler.authzr[self.doms[2]] = acme_util.gen_authzr( + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) self.chall_update = {} - for dom in self.doms: - self.chall_update[dom] = [ - challb_to_achall(challb, mock.Mock(key="dummy_key"), dom) - for challb in self.handler.authzr[dom].body.challenges] + for i, aauthzr in enumerate(self.handler.aauthzrs): + self.chall_update[i] = [ + challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) + for challb in aauthzr.authzr.body.challenges] @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid self.handler._poll_challenges(self.chall_update, False) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_VALID) + for aauthzr in self.handler.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) @mock.patch("certbot.auth_handler.time") def test_poll_challenges_failure_best_effort(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid self.handler._poll_challenges(self.chall_update, True) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_PENDING) + for aauthzr in self.handler.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) @mock.patch("certbot.auth_handler.time") @test_util.patch_get_utility() @@ -345,7 +343,7 @@ class PollChallengesTest(unittest.TestCase): def test_unable_to_find_challenge_status(self, unused_mock_time): from certbot.auth_handler import challb_to_achall self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.chall_update[self.doms[0]].append( + self.chall_update[0].append( challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index c5935d722..1bba6991a 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -426,6 +426,10 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods namespace = self.parse(["--no-delete-after-revoke"]) self.assertFalse(namespace.delete_after_revoke) + def test_allow_subset_with_wildcard(self): + self.assertRaises(errors.Error, self.parse, + "--allow-subset-of-names -d *.example.org".split()) + class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index ed9c140e7..b51275d9e 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -222,6 +222,7 @@ class ClientTest(ClientTestCommon): mock.sentinel.chain) authzr = self._authzr_from_domains(["example.com"]) + self.config.allow_subset_of_names = True self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 50d323ffd..0e280f3ab 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -487,6 +487,26 @@ class EnforceDomainSanityTest(unittest.TestCase): self._call('this.is.xn--ls8h.tld') +class IsWildcardDomainTest(unittest.TestCase): + """Tests for is_wildcard_domain.""" + + def setUp(self): + self.wildcard = u"*.example.org" + self.no_wildcard = u"example.org" + + def _call(self, domain): + from certbot.util import is_wildcard_domain + return is_wildcard_domain(domain) + + def test_no_wildcard(self): + self.assertFalse(self._call(self.no_wildcard)) + self.assertFalse(self._call(self.no_wildcard.encode())) + + def test_wildcard(self): + self.assertTrue(self._call(self.wildcard)) + self.assertTrue(self._call(self.wildcard.encode())) + + class OsInfoTest(unittest.TestCase): """Test OS / distribution detection""" diff --git a/certbot/util.py b/certbot/util.py index f47c5da9c..f7ce6a3bc 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -601,6 +601,24 @@ def enforce_domain_sanity(domain): return domain +def is_wildcard_domain(domain): + """"Is domain a wildcard domain? + + :param damain: domain to check + :type domain: `bytes` or `str` or `unicode` + + :returns: True if domain is a wildcard, otherwise, False + :rtype: bool + + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + + return domain.startswith(wildcard_marker) + + def get_strict_version(normalized): """Converts a normalized version to a strict version. From e9bc4a319b9989a300dd574466a15edd581ee3c4 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 28 Feb 2018 21:31:47 +0200 Subject: [PATCH 95/98] Apache plugin wildcard support for ACMEv2 (#5608) In `deploy_cert()` and `enhance()`, the user will be presented with a dialog to choose from the VirtualHosts that can be covered by the wildcard domain name. The (multiple) selection result will then be handled in a similar way that we previously handled a single VirtualHost that was returned by the `_find_best_vhost()`. Additionally the selected VirtualHosts are added to a dictionary that maps selections to a wildcard domain to be reused in the later `enhance()` call and not forcing the user to select the same VirtualHosts again. * Apache plugin wildcard support * Present dialog only once per domain, added tests * Raise exception if no VHosts selected for wildcard domain --- certbot-apache/certbot_apache/configurator.py | 141 +++++++++++++++++- certbot-apache/certbot_apache/display_ops.py | 48 +++++- certbot-apache/certbot_apache/obj.py | 13 ++ .../certbot_apache/tests/configurator_test.py | 100 +++++++++++++ .../certbot_apache/tests/display_ops_test.py | 30 ++++ certbot/tests/main_test.py | 1 - 6 files changed, 316 insertions(+), 17 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 4bb2cbebd..6377bb114 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -5,6 +5,7 @@ import logging import os import pkg_resources import re +import six import socket import time @@ -152,6 +153,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc = dict() # Outstanding challenges self._chall_out = set() + # List of vhosts configured per wildcard domain on this run. + # used by deploy_cert() and enhance() + self._wildcard_vhosts = dict() # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) @@ -262,6 +266,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.aug, self.conf("server-root"), self.conf("vhost-root"), self.version, configurator=self) + def _wildcard_domain(self, domain): + """ + Checks if domain is a wildcard domain + + :param str domain: Domain to check + + :returns: If the domain is wildcard domain + :rtype: bool + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + return domain.startswith(wildcard_marker) + def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): """Deploys certificate to specified virtual host. @@ -280,9 +299,112 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): a lack of directives """ - # Choose vhost before (possible) enabling of mod_ssl, to keep the - # vhost choice namespace similar with the pre-validation one. - vhost = self.choose_vhost(domain) + vhosts = self.choose_vhosts(domain) + for vhost in vhosts: + self._deploy_cert(vhost, cert_path, key_path, chain_path, fullchain_path) + + def choose_vhosts(self, domain, create_if_no_ssl=True): + """ + Finds VirtualHosts that can be used with the provided domain + + :param str domain: Domain name to match VirtualHosts to + :param bool create_if_no_ssl: If found VirtualHost doesn't have a HTTPS + counterpart, should one get created + + :returns: List of VirtualHosts or None + :rtype: `list` of :class:`~certbot_apache.obj.VirtualHost` + """ + + if self._wildcard_domain(domain): + if domain in self._wildcard_vhosts: + # Vhosts for a wildcard domain were already selected + return self._wildcard_vhosts[domain] + # Ask user which VHosts to support. + # Returned objects are guaranteed to be ssl vhosts + return self._choose_vhosts_wildcard(domain, create_if_no_ssl) + else: + return [self.choose_vhost(domain)] + + def _vhosts_for_wildcard(self, domain): + """ + Get VHost objects for every VirtualHost that the user wants to handle + with the wildcard certificate. + """ + + # Collect all vhosts that match the name + matched = set() + for vhost in self.vhosts: + for name in vhost.get_names(): + if self._in_wildcard_scope(name, domain): + matched.add(vhost) + + return list(matched) + + def _in_wildcard_scope(self, name, domain): + """ + Helper method for _vhosts_for_wildcard() that makes sure that the domain + is in the scope of wildcard domain. + + eg. in scope: domain = *.wild.card, name = 1.wild.card + not in scope: domain = *.wild.card, name = 1.2.wild.card + """ + if len(name.split(".")) == len(domain.split(".")): + return fnmatch.fnmatch(name, domain) + + + def _choose_vhosts_wildcard(self, domain, create_ssl=True): + """Prompts user to choose vhosts to install a wildcard certificate for""" + + # Get all vhosts that are covered by the wildcard domain + vhosts = self._vhosts_for_wildcard(domain) + + # Go through the vhosts, making sure that we cover all the names + # present, but preferring the SSL vhosts + filtered_vhosts = dict() + for vhost in vhosts: + for name in vhost.get_names(): + if vhost.ssl: + # Always prefer SSL vhosts + filtered_vhosts[name] = vhost + elif name not in filtered_vhosts and create_ssl: + # Add if not in list previously + filtered_vhosts[name] = vhost + + # Only unique VHost objects + dialog_input = set([vhost for vhost in filtered_vhosts.values()]) + + # Ask the user which of names to enable, expect list of names back + dialog_output = display_ops.select_vhost_multiple(list(dialog_input)) + + if not dialog_output: + logger.error( + "No vhost exists with servername or alias for domain %s. " + "No vhost was selected. Please specify ServerName or ServerAlias " + "in the Apache config.", + domain) + raise errors.PluginError("No vhost selected") + + # Make sure we create SSL vhosts for the ones that are HTTP only + # if requested. + return_vhosts = list() + for vhost in dialog_output: + if not vhost.ssl: + return_vhosts.append(self.make_vhost_ssl(vhost)) + else: + return_vhosts.append(vhost) + + self._wildcard_vhosts[domain] = return_vhosts + return return_vhosts + + + def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): + """ + Helper function for deploy_cert() that handles the actual deployment + this exists because we might want to do multiple deployments per + domain originally passed for deploy_cert(). This is especially true + with wildcard certificates + """ + # This is done first so that ssl module is enabled and cert_path, # cert_key... can all be parsed appropriately @@ -311,7 +433,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "Unable to find cert and/or key directives") - logger.info("Deploying Certificate for %s to VirtualHost %s", domain, vhost.filep) + logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) if self.version < (2, 4, 8) or (chain_path and not fullchain_path): # install SSLCertificateFile, SSLCertificateKeyFile, @@ -327,8 +449,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "version of Apache") else: if not fullchain_path: - raise errors.PluginError("Please provide the --fullchain-path\ - option pointing to your full chain file") + raise errors.PluginError("Please provide the --fullchain-path " + "option pointing to your full chain file") set_cert_path = fullchain_path self.aug.set(path["cert_path"][-1], fullchain_path) self.aug.set(path["cert_key"][-1], key_path) @@ -391,7 +513,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.error( "No vhost exists with servername or alias of %s. " "No vhost was selected. Please specify ServerName or ServerAlias " - "in the Apache config, or split vhosts into separate files.", + "in the Apache config.", target_name) raise errors.PluginError("No vhost selected") elif temp: @@ -1376,8 +1498,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): except KeyError: raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) + + vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) try: - func(self.choose_vhost(domain), options) + for vhost in vhosts: + func(vhost, options) except errors.PluginError: logger.warning("Failed %s for %s", enhancement, domain) raise diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index 9529c1ab3..097b84b96 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -13,10 +13,44 @@ import certbot.display.util as display_util logger = logging.getLogger(__name__) +def select_vhost_multiple(vhosts): + """Select multiple Vhosts to install the certificate for + + :param vhosts: Available Apache VirtualHosts + :type vhosts: :class:`list` of type `~obj.Vhost` + + :returns: List of VirtualHosts + :rtype: :class:`list`of type `~obj.Vhost` + """ + if not vhosts: + return list() + tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] + # Remove the extra newline from the last entry + if len(tags_list): + tags_list[-1] = tags_list[-1][:-1] + code, names = zope.component.getUtility(interfaces.IDisplay).checklist( + "Which VirtualHosts would you like to install the wildcard certificate for?", + tags=tags_list, force_interactive=True) + if code == display_util.OK: + return_vhosts = _reversemap_vhosts(names, vhosts) + return return_vhosts + return [] + +def _reversemap_vhosts(names, vhosts): + """Helper function for select_vhost_multiple for mapping string + representations back to actual vhost objects""" + return_vhosts = list() + + for selection in names: + for vhost in vhosts: + if vhost.display_repr().strip() == selection.strip(): + return_vhosts.append(vhost) + return return_vhosts + def select_vhost(domain, vhosts): """Select an appropriate Apache Vhost. - :param vhosts: Available Apache Virtual Hosts + :param vhosts: Available Apache VirtualHosts :type vhosts: :class:`list` of type `~obj.Vhost` :returns: VirtualHost or `None` @@ -25,13 +59,11 @@ def select_vhost(domain, vhosts): """ if not vhosts: return None - while True: - code, tag = _vhost_menu(domain, vhosts) - if code == display_util.OK: - return vhosts[tag] - else: - return None - + code, tag = _vhost_menu(domain, vhosts) + if code == display_util.OK: + return vhosts[tag] + else: + return None def _vhost_menu(domain, vhosts): """Select an appropriate Apache Vhost. diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index 1e3579858..fcf3bfe08 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -167,6 +167,19 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods active="Yes" if self.enabled else "No", modmacro="Yes" if self.modmacro else "No")) + def display_repr(self): + """Return a representation of VHost to be used in dialog""" + return ( + "File: {filename}\n" + "Addresses: {addrs}\n" + "Names: {names}\n" + "HTTPS: {https}\n".format( + filename=self.filep, + addrs=", ".join(str(addr) for addr in self.addrs), + names=", ".join(self.get_names()), + https="Yes" if self.ssl else "No")) + + def __eq__(self, other): if isinstance(other, self.__class__): return (self.filep == other.filep and self.path == other.path and diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 8f34d33d3..c9bf9a63f 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1337,6 +1337,106 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enable_mod, "whatever") + def test_wildcard_domain(self): + # pylint: disable=protected-access + cases = {u"*.example.org": True, b"*.x.example.org": True, + u"a.example.org": False, b"a.x.example.org": False} + for key in cases.keys(): + self.assertEqual(self.config._wildcard_domain(key), cases[key]) + + def test_choose_vhosts_wildcard(self): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[3]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=True) + # Check that the dialog was called with one vh: certbot.demo + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[3]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].name == "certbot.demo") + self.assertTrue(vhs[0].ssl) + + self.assertFalse(vhs[0] == self.vh_truth[3]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_no_ssl(self, mock_makessl): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[1]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=False) + self.assertFalse(mock_makessl.called) + self.assertEquals(vhs[0], self.vh_truth[1]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._vhosts_for_wildcard") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_already_ssl(self, mock_makessl, mock_vh_for_w): + # pylint: disable=protected-access + # Already SSL vhost + mock_vh_for_w.return_value = [self.vh_truth[7]] + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[7]] + vhs = self.config._choose_vhosts_wildcard("whatever", + create_ssl=True) + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[7]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + # Ensure that make_vhost_ssl was not called, vhost.ssl == true + self.assertFalse(mock_makessl.called) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].ssl) + self.assertEquals(vhs[0], self.vh_truth[7]) + + + def test_deploy_cert_wildcard(self): + # pylint: disable=protected-access + mock_choose_vhosts = mock.MagicMock() + mock_choose_vhosts.return_value = [self.vh_truth[7]] + self.config._choose_vhosts_wildcard = mock_choose_vhosts + mock_d = "certbot_apache.configurator.ApacheConfigurator._deploy_cert" + with mock.patch(mock_d) as mock_dep: + self.config.deploy_cert("*.wildcard.example.org", "/tmp/path", + "/tmp/path", "/tmp/path", "/tmp/path") + self.assertTrue(mock_dep.called) + self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(self.vh_truth[7], mock_dep.call_args_list[0][0][0]) + + @mock.patch("certbot_apache.display_ops.select_vhost_multiple") + def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog): + # pylint: disable=protected-access + mock_dialog.return_value = [] + self.assertRaises(errors.PluginError, + self.config.deploy_cert, + "*.wild.cat", "/tmp/path", "/tmp/path", + "/tmp/path", "/tmp/path") + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_after_install(self, mock_choose): + # pylint: disable=protected-access + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertFalse(mock_choose.called) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_no_install(self, mock_choose): + mock_choose.return_value = [self.vh_truth[3]] + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertTrue(mock_choose.called) + + class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" # pylint: disable=protected-access diff --git a/certbot-apache/certbot_apache/tests/display_ops_test.py b/certbot-apache/certbot_apache/tests/display_ops_test.py index e59d411bd..df5cdbac0 100644 --- a/certbot-apache/certbot_apache/tests/display_ops_test.py +++ b/certbot-apache/certbot_apache/tests/display_ops_test.py @@ -11,9 +11,39 @@ from certbot.tests import util as certbot_util from certbot_apache import obj +from certbot_apache.display_ops import select_vhost_multiple from certbot_apache.tests import util +class SelectVhostMultiTest(unittest.TestCase): + """Tests for certbot_apache.display_ops.select_vhost_multiple.""" + + def setUp(self): + self.base_dir = "/example_path" + self.vhosts = util.get_vh_truth( + self.base_dir, "debian_apache_2_4/multiple_vhosts") + + def test_select_no_input(self): + self.assertFalse(select_vhost_multiple([])) + + @certbot_util.patch_get_utility() + def test_select_correct(self, mock_util): + mock_util().checklist.return_value = ( + display_util.OK, [self.vhosts[3].display_repr(), + self.vhosts[2].display_repr()]) + vhs = select_vhost_multiple([self.vhosts[3], + self.vhosts[2], + self.vhosts[1]]) + self.assertTrue(self.vhosts[2] in vhs) + self.assertTrue(self.vhosts[3] in vhs) + self.assertFalse(self.vhosts[1] in vhs) + + @certbot_util.patch_get_utility() + def test_select_cancel(self, mock_util): + mock_util().checklist.return_value = (display_util.CANCEL, "whatever") + vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]]) + self.assertFalse(vhs) + class SelectVhostTest(unittest.TestCase): """Tests for certbot_apache.display_ops.select_vhost.""" diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index c31a3fb33..b778f05ea 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -940,7 +940,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertRaises(errors.ConfigurationError, self._call, ['-d', (('a' * 50) + '.') * 10]) - # Bare IP address (this is actually a different error message now) self.assertRaises(errors.ConfigurationError, self._call, From 78735fa2c3ebaee6c7aba02bbf939597a9075cbb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 28 Feb 2018 16:08:06 -0800 Subject: [PATCH 96/98] Suggest DNS authenticator when it's needed (#5638) --- certbot/auth_handler.py | 16 ++++++++++++---- certbot/tests/auth_handler_test.py | 6 ++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 2b38e4af5..67d36c8cc 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -433,7 +433,7 @@ def _find_smart_path(challbs, preferences, combinations): combo_total = 0 if not best_combo: - _report_no_chall_path() + _report_no_chall_path(challbs) return best_combo @@ -454,15 +454,23 @@ def _find_dumb_path(challbs, preferences): if supported: path.append(i) else: - _report_no_chall_path() + _report_no_chall_path(challbs) return path -def _report_no_chall_path(): - """Logs and raises an error that no satisfiable chall path exists.""" +def _report_no_chall_path(challbs): + """Logs and raises an error that no satisfiable chall path exists. + + :param challbs: challenges from the authorization that can't be satisfied + + """ msg = ("Client with the currently selected authenticator does not support " "any combination of challenges that will satisfy the CA.") + if len(challbs) == 1 and isinstance(challbs[0].chall, challenges.DNS01): + msg += ( + " You may need to use an authenticator " + "plugin that can do challenges over DNS.") logger.fatal(msg) raise errors.AuthorizationError(msg) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 7650d2c95..b6af3d0f5 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -272,6 +272,12 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_net.acme_version = 2 self._test_preferred_challenges_not_supported_common(combos=False) + def test_dns_only_challenge_not_supported(self): + authzrs = [gen_dom_authzr(domain="0", challs=[acme_util.DNS01])] + mock_order = mock.MagicMock(authorizations=authzrs) + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def _validate_all(self, unused_1, unused_2): for i, aauthzr in enumerate(self.handler.aauthzrs): azr = aauthzr.authzr From 38d5144fff79e10507ee7d53cb6664c8180d2245 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 08:25:32 -0800 Subject: [PATCH 97/98] Drop min coverage to 63 (#5641) --- tests/boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 24d224cb0..ea412b6b9 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -437,4 +437,4 @@ then . ./certbot-nginx/tests/boulder-integration.sh fi -coverage report --fail-under 64 -m +coverage report --fail-under 63 -m From 559220c2eff90975fb671945d780c2a757d0e167 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 10:11:15 -0800 Subject: [PATCH 98/98] Add basic ACMEv2 integration tests (#5635) * Use newer boulder config * Use ACMEv2 endpoint if requested * Add v2 integration tests * Work with unset variables * Add wildcard issuance test * quote domains --- .travis.yml | 6 +++++- tests/boulder-fetch.sh | 7 +++++++ tests/boulder-integration.sh | 6 ++++++ tests/integration/_common.sh | 5 +++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 42b8d679d..c62664180 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,11 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=1 + env: TOXENV=py27_install BOULDER_INTEGRATION=v1 + sudo: required + services: docker + - python: "2.7" + env: TOXENV=py27_install BOULDER_INTEGRATION=v2 sudo: required services: docker - python: "2.7" diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 08eb736c2..fc9cbaae7 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -16,6 +16,13 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml + +# If we're testing against ACMEv2, we need to use a newer boulder config for +# now. See https://github.com/letsencrypt/boulder#quickstart. +if [ "$BOULDER_INTEGRATION" = "v2" ]; then + sed -i 's/BOULDER_CONFIG_DIR: .*/BOULDER_CONFIG_DIR: test\/config-next/' docker-compose.yml +fi + docker-compose up -d set +x # reduce verbosity while waiting for boulder diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ea412b6b9..f2b0dcf60 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -430,6 +430,12 @@ for path in $archive $conf $live; do fi done +# Test ACMEv2-only features +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then + common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ + --manual-auth-hook ./tests/manual-dns-auth.sh +fi + # Most CI systems set this variable to true. # If the tests are running as part of CI, Nginx should be available. if ${CI:-false} || type nginx; diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index d151bdc3f..236090a14 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,6 +16,11 @@ certbot_test () { "$@" } +# Use local ACMEv2 endpoint if requested and SERVER isn't already set. +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" -a -z "${SERVER:+x}" ]; then + SERVER="http://localhost:4001/directory" +fi + certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" omit_patterns="$omit_patterns,*_test.py,*_test_*,"