From 2bc64183a8fe963959e91bfd87fb8f0e64b17650 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 11 Nov 2019 17:11:47 -0800 Subject: [PATCH 01/72] fix docstring --- certbot/plugins/common_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 977500f86..5d529e993 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -178,7 +178,7 @@ class InstallerTest(test_util.ConfigTestCase): class AddrTest(unittest.TestCase): - """Tests for certbot._internal.client.plugins.common.Addr.""" + """Tests for certbot._internal.plugins.common.Addr.""" def setUp(self): from certbot.plugins.common import Addr From 86926dff9293ab24d4f025ccf70c47619750d897 Mon Sep 17 00:00:00 2001 From: OsirisInferi Date: Tue, 4 Feb 2020 19:27:27 +0100 Subject: [PATCH 02/72] Use unrestrictive umask for challenge directory --- certbot-apache/certbot_apache/_internal/http_01.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index c34abc2b4..53ccd2bc7 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -168,7 +168,9 @@ class ApacheHttp01(common.ChallengePerformer): def _set_up_challenges(self): if not os.path.isdir(self.challenge_dir): + old_umask = os.umask(0o022) filesystem.makedirs(self.challenge_dir, 0o755) + os.umask(old_umask) responses = [] for achall in self.achalls: From 601a114d1ba6030f3f765ff86bb39658172e0a75 Mon Sep 17 00:00:00 2001 From: OsirisInferi Date: Tue, 4 Feb 2020 19:47:27 +0100 Subject: [PATCH 03/72] Update changelog --- certbot/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 86d27143c..01cd3d402 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -19,6 +19,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed * Fix collections.abc imports for Python 3.9. +* Fix Apache plugin to use less restrictive umask for making the challenge directory when a restrictive umask was set when certbot was started. More details about these changes can be found on our GitHub repo. From f3ed13374456f3b53fc87dc0fa1ed71b1efa37e7 Mon Sep 17 00:00:00 2001 From: OsirisInferi Date: Wed, 5 Feb 2020 22:17:29 +0100 Subject: [PATCH 04/72] Wrap makedirs() within exception handelrs --- certbot-apache/certbot_apache/_internal/http_01.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 53ccd2bc7..ad62a77bb 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -169,8 +169,14 @@ class ApacheHttp01(common.ChallengePerformer): def _set_up_challenges(self): if not os.path.isdir(self.challenge_dir): old_umask = os.umask(0o022) - filesystem.makedirs(self.challenge_dir, 0o755) - os.umask(old_umask) + try: + filesystem.makedirs(self.challenge_dir, 0o755) + except OSError as exception: + if exception.errno not in (errno.EEXIST, errno.EISDIR): + raise errors.PluginError( + "Couldn't create root for http-01 challenge") + finally: + os.umask(old_umask) responses = [] for achall in self.achalls: From d3a4b8fd8c068624b40179f567e191b6979bf6cf Mon Sep 17 00:00:00 2001 From: OsirisInferi Date: Wed, 5 Feb 2020 22:27:12 +0100 Subject: [PATCH 05/72] Missing import --- certbot-apache/certbot_apache/_internal/http_01.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index ad62a77bb..6c822cc38 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -1,5 +1,6 @@ """A class that performs HTTP-01 challenges for Apache""" import logging +import errno from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module From df584a3b90f9c18efaac07a5d7fb0bf9be5a81a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 12 Feb 2020 13:12:03 -0800 Subject: [PATCH 06/72] Remove _internal from docstring. --- certbot/tests/plugins/common_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index fc05bf894..7543f28f3 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -177,7 +177,7 @@ class InstallerTest(test_util.ConfigTestCase): class AddrTest(unittest.TestCase): - """Tests for certbot._internal.plugins.common.Addr.""" + """Tests for certbot.plugins.common.Addr.""" def setUp(self): from certbot.plugins.common import Addr From 9819443440382695b74b77379d76e4886c0bdf70 Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Sat, 22 Feb 2020 15:22:27 +0100 Subject: [PATCH 07/72] Add test --- certbot-apache/tests/http_01_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 643a6bdd5..422a76443 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -1,5 +1,6 @@ """Test for certbot_apache._internal.http_01.""" import unittest +import errno import mock @@ -197,6 +198,12 @@ class ApacheHttp01Test(util.ApacheTest): self.assertTrue(os.path.exists(challenge_dir)) + @mock.patch("certbot_apache._internal.http_01.filesystem.makedirs") + def test_failed_makedirs(self, mock_makedirs): + mock_makedirs.side_effect = OSError(errno.EACCES, "msg") + self.http.add_chall(self.achalls[0]) + self.assertRaises(errors.PluginError, self.http.perform) + def _test_challenge_conf(self): with open(self.http.challenge_conf_pre) as f: pre_conf_contents = f.read() From d6ef34a03e21f8e8da0bc45d6b7124d50cfc5ec1 Mon Sep 17 00:00:00 2001 From: cumul Date: Tue, 19 Dec 2017 05:34:31 +0900 Subject: [PATCH 08/72] Use UTF-8 encoding for nginx plugin --- certbot-nginx/certbot_nginx/_internal/parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index f71d7c018..38c4d63e1 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -1,5 +1,6 @@ """NginxParser is a member object of the NginxConfigurator class.""" import copy +import codecs import functools import glob import logging @@ -205,7 +206,7 @@ class NginxParser(object): if item in self.parsed and not override: continue try: - with open(item) as _file: + with codecs.open(item, "r", "utf-8") as _file: parsed = nginxparser.load(_file) self.parsed[item] = parsed trees.append(parsed) @@ -414,7 +415,7 @@ class NginxParser(object): def _parse_ssl_options(ssl_options): if ssl_options is not None: try: - with open(ssl_options) as _file: + with codecs.open(ssl_options, "r", "utf-8") as _file: return nginxparser.load(_file) except IOError: logger.warning("Missing NGINX TLS options file: %s", ssl_options) From 247d9cd8870635dd9d065883e7853b86c0c76531 Mon Sep 17 00:00:00 2001 From: cumul Date: Wed, 31 Oct 2018 15:45:30 +0900 Subject: [PATCH 09/72] Use `io` module instead of `codecs` See https://mail.python.org/pipermail/python-list/2015-March/687124.html --- certbot-nginx/certbot_nginx/_internal/parser.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 38c4d63e1..d6182870c 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -1,8 +1,8 @@ """NginxParser is a member object of the NginxConfigurator class.""" import copy -import codecs import functools import glob +import io import logging import re @@ -206,12 +206,14 @@ class NginxParser(object): if item in self.parsed and not override: continue try: - with codecs.open(item, "r", "utf-8") as _file: + with io.open(item, "r", encoding="utf-8") as _file: parsed = nginxparser.load(_file) self.parsed[item] = parsed trees.append(parsed) except IOError: logger.warning("Could not open file: %s", item) + except UnicodeDecodeError: + logger.warning("Could not read file: %s due to invalid unicode character. Only UTF-8 encoding is supported.", item) except pyparsing.ParseException as err: logger.debug("Could not parse file: %s due to %s", item, err) return trees @@ -415,10 +417,12 @@ class NginxParser(object): def _parse_ssl_options(ssl_options): if ssl_options is not None: try: - with codecs.open(ssl_options, "r", "utf-8") as _file: + with io.open(ssl_options, "r", encoding="utf-8") as _file: return nginxparser.load(_file) except IOError: logger.warning("Missing NGINX TLS options file: %s", ssl_options) + except UnicodeDecodeError: + logger.warn("Could not read file: %s due to invalid unicode character. Only UTF-8 encoding is supported.", ssl_options) except pyparsing.ParseBaseException as err: logger.debug("Could not parse file: %s due to %s", ssl_options, err) return [] From 8b90b555186c0c51b0c37389375afcda2f27a3d6 Mon Sep 17 00:00:00 2001 From: cumul Date: Wed, 31 Oct 2018 16:48:01 +0900 Subject: [PATCH 10/72] Added test for valid/invalid unicode characters --- certbot-nginx/tests/parser_test.py | 12 ++++++++++++ .../testdata/etc_nginx/invalid_unicode_comments.conf | 7 +++++++ .../testdata/etc_nginx/valid_unicode_comments.conf | 9 +++++++++ 3 files changed, 28 insertions(+) create mode 100644 certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf create mode 100644 certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index f3a5665c5..232346396 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -482,6 +482,18 @@ class NginxParserTest(util.NginxTest): called = True self.assertTrue(called) + def test_valid_unicode_characters(self): + nparser = parser.NginxParser(self.config_path) + # pylint: disable=protected-access + parsed = nparser._parse_files(nparser.abs_path('unicode_support/valid_unicode_comments.conf')) + self.assertEqual(['server'], parsed[0][2][0]) + self.assertEqual(['listen', '80'], parsed[0][2][1][3]) + + def test_invalid_unicode_characters(self): + nparser = parser.NginxParser(self.config_path) + # pylint: disable=protected-access + parsed = nparser._parse_files(nparser.abs_path('unicode_support/invalid_unicode_comments.conf')) + self.assertEqual([], parsed) if __name__ == "__main__": diff --git a/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf b/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf new file mode 100644 index 000000000..596044cc9 --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf @@ -0,0 +1,7 @@ +# This configuration file is saved with EUC-KR (a.k.a. cp949) encoding, +# including some Korean alphabets. + +server { + # 안녕하세요. 80번 포트에서 요청을 기다린다. + listen 80; +} diff --git a/certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf b/certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf new file mode 100644 index 000000000..89c978b2e --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf @@ -0,0 +1,9 @@ +# This configuration file is saved with valid UTF-8 encoding, +# including some CJK alphabets. + +server { + # �븞�뀞�븯�꽭�슂. 80踰� �룷�듃�뿉�꽌 �슂泥��쓣 湲곕떎由곕떎. + # �걪�굯�겓�걾�겘��귨폍竊먪빁�깮�꺖�깉�걢�굢�꺁�궚�궓�궧�깉�굮孃끹겇��� + # 鵝졾�썬�귞춬孃끿ク�룭80訝딁쉪瑥룡콆��� + listen 80; +} From 0b21e716cabba503e4ec7ce7e722e93a319f361f Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 30 Jan 2019 15:12:55 -0800 Subject: [PATCH 11/72] Fix lint problems with long lines --- certbot-nginx/certbot_nginx/_internal/parser.py | 7 +++++-- certbot-nginx/tests/parser_test.py | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index d6182870c..72d4d38de 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -213,7 +213,9 @@ class NginxParser(object): except IOError: logger.warning("Could not open file: %s", item) except UnicodeDecodeError: - logger.warning("Could not read file: %s due to invalid unicode character. Only UTF-8 encoding is supported.", item) + logger.warning("Could not read file: %s due to invalid " + "character. Only UTF-8 encoding is " + "supported.", item) except pyparsing.ParseException as err: logger.debug("Could not parse file: %s due to %s", item, err) return trees @@ -422,7 +424,8 @@ def _parse_ssl_options(ssl_options): except IOError: logger.warning("Missing NGINX TLS options file: %s", ssl_options) except UnicodeDecodeError: - logger.warn("Could not read file: %s due to invalid unicode character. Only UTF-8 encoding is supported.", ssl_options) + logger.warn("Could not read file: %s due to invalid character. " + "Only UTF-8 encoding is supported.", ssl_options) except pyparsing.ParseBaseException as err: logger.debug("Could not parse file: %s due to %s", ssl_options, err) return [] diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 232346396..632ea179f 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -485,14 +485,16 @@ class NginxParserTest(util.NginxTest): def test_valid_unicode_characters(self): nparser = parser.NginxParser(self.config_path) # pylint: disable=protected-access - parsed = nparser._parse_files(nparser.abs_path('unicode_support/valid_unicode_comments.conf')) + path = nparser.abs_path('unicode_support/valid_unicode_comments.conf') + parsed = nparser._parse_files(path) self.assertEqual(['server'], parsed[0][2][0]) self.assertEqual(['listen', '80'], parsed[0][2][1][3]) def test_invalid_unicode_characters(self): nparser = parser.NginxParser(self.config_path) # pylint: disable=protected-access - parsed = nparser._parse_files(nparser.abs_path('unicode_support/invalid_unicode_comments.conf')) + path = nparser.abs_path('unicode_support/invalid_unicode_comments.conf') + parsed = nparser._parse_files(path) self.assertEqual([], parsed) From c3cfd412c91cfdc2d0ba95ad4c3751324bf8b132 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 01:46:41 +0900 Subject: [PATCH 12/72] Relpace deprecated `logger.warn()` with `logger.warning()` --- certbot-nginx/certbot_nginx/_internal/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 72d4d38de..0c1151826 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -424,8 +424,8 @@ def _parse_ssl_options(ssl_options): except IOError: logger.warning("Missing NGINX TLS options file: %s", ssl_options) except UnicodeDecodeError: - logger.warn("Could not read file: %s due to invalid character. " - "Only UTF-8 encoding is supported.", ssl_options) + logger.warning("Could not read file: %s due to invalid character. " + "Only UTF-8 encoding is supported.", ssl_options) except pyparsing.ParseBaseException as err: logger.debug("Could not parse file: %s due to %s", ssl_options, err) return [] From 22685ef86fa2f00be647403f3c64cec79b4d2be1 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:23:46 +0900 Subject: [PATCH 13/72] Remove `unicode_support/` path in test case --- certbot-nginx/tests/parser_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 632ea179f..d766a6a9b 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -485,7 +485,7 @@ class NginxParserTest(util.NginxTest): def test_valid_unicode_characters(self): nparser = parser.NginxParser(self.config_path) # pylint: disable=protected-access - path = nparser.abs_path('unicode_support/valid_unicode_comments.conf') + path = nparser.abs_path('valid_unicode_comments.conf') parsed = nparser._parse_files(path) self.assertEqual(['server'], parsed[0][2][0]) self.assertEqual(['listen', '80'], parsed[0][2][1][3]) @@ -493,7 +493,7 @@ class NginxParserTest(util.NginxTest): def test_invalid_unicode_characters(self): nparser = parser.NginxParser(self.config_path) # pylint: disable=protected-access - path = nparser.abs_path('unicode_support/invalid_unicode_comments.conf') + path = nparser.abs_path('invalid_unicode_comments.conf') parsed = nparser._parse_files(path) self.assertEqual([], parsed) From 36311a276b7ffbff9bcc8b019f6a12f97e153712 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:46:27 +0900 Subject: [PATCH 14/72] Add test case for `_parse_ssl_options()` --- certbot-nginx/tests/parser_test.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index d766a6a9b..a0484f965 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -497,6 +497,25 @@ class NginxParserTest(util.NginxTest): parsed = nparser._parse_files(path) self.assertEqual([], parsed) + def test_valid_unicode_characters_in_ssl_options(self): + nparser = parser.NginxParser(self.config_path) + path = nparser.abs_path('valid_unicode_comments.conf') + parsed = parser._parse_ssl_options(path) # pylint: disable=protected-access + self.assertEqual(['server'], parsed[2][0]) + self.assertEqual(['listen', '80'], parsed[2][1][3]) + + def test_invalid_unicode_characters_in_ssl_options(self): + with self.assertLogs() as log: + nparser = parser.NginxParser(self.config_path) + path = nparser.abs_path('invalid_unicode_comments.conf') + parsed = parser._parse_ssl_options(path) # pylint: disable=protected-access + + self.assertEqual([], parsed) + self.assertTrue([ + True + for output in log.output + if ('invalid character' in output) and ('UTF-8' in output) + ]) if __name__ == "__main__": unittest.main() # pragma: no cover From 20df5507ae9a17df35f9c1152a34d0c52ba304f6 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:47:44 +0900 Subject: [PATCH 15/72] Add logging test for `_parse_files()` --- certbot-nginx/tests/parser_test.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index a0484f965..89bcf689a 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -491,11 +491,17 @@ class NginxParserTest(util.NginxTest): self.assertEqual(['listen', '80'], parsed[0][2][1][3]) def test_invalid_unicode_characters(self): - nparser = parser.NginxParser(self.config_path) - # pylint: disable=protected-access - path = nparser.abs_path('invalid_unicode_comments.conf') - parsed = nparser._parse_files(path) + with self.assertLogs() as log: + nparser = parser.NginxParser(self.config_path) + path = nparser.abs_path('invalid_unicode_comments.conf') + parsed = nparser._parse_files(path) # pylint: disable=protected-access + self.assertEqual([], parsed) + self.assertTrue([ + True + for output in log.output + if ('invalid character' in output) and ('UTF-8' in output) + ]) def test_valid_unicode_characters_in_ssl_options(self): nparser = parser.NginxParser(self.config_path) From 2aac24c9829d565d6ba6aa42ad9fc7e5ba4926b3 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:48:21 +0900 Subject: [PATCH 16/72] Trivial code clean-up --- certbot-nginx/tests/parser_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 89bcf689a..1f9e3c996 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -484,9 +484,8 @@ class NginxParserTest(util.NginxTest): def test_valid_unicode_characters(self): nparser = parser.NginxParser(self.config_path) - # pylint: disable=protected-access path = nparser.abs_path('valid_unicode_comments.conf') - parsed = nparser._parse_files(path) + parsed = nparser._parse_files(path) # pylint: disable=protected-access self.assertEqual(['server'], parsed[0][2][0]) self.assertEqual(['listen', '80'], parsed[0][2][1][3]) From b3071aab29c480d52837e70fcf3fed9379f22200 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:52:41 +0900 Subject: [PATCH 17/72] Add my name to AUTHORS.md :) --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 80a24d3be..8653382b8 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -268,3 +268,4 @@ Authors * [YourDaddyIsHere](https://github.com/YourDaddyIsHere) * [Zach Shepherd](https://github.com/zjs) * [�솃訝�](https://github.com/chenxsan) +* [Yuseong Cho](https://github.com/g6123) From d68f37ae88d32dcca97785df9ec4c3cf70c437ae Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:56:43 +0900 Subject: [PATCH 18/72] Add this change to CHANGELOG.md --- certbot/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 126b07eec..837f48db2 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -17,7 +17,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* +* Fix nginx plguin crash when non-ASCII configuration file is being read More details about these changes can be found on our GitHub repo. From 32904d8c9e2c1c6dd253eb860472053053238238 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 04:26:37 +0900 Subject: [PATCH 19/72] Add `TestCase.assertLogs()` backport for Python 2.7 --- certbot-nginx/tests/test_log_util.py | 123 +++++++++++++++++++++++++++ certbot-nginx/tests/test_util.py | 3 +- 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 certbot-nginx/tests/test_log_util.py diff --git a/certbot-nginx/tests/test_log_util.py b/certbot-nginx/tests/test_log_util.py new file mode 100644 index 000000000..f8135ca26 --- /dev/null +++ b/certbot-nginx/tests/test_log_util.py @@ -0,0 +1,123 @@ +"""Backport for `TestCase.assertLogs()`. + +Most of the idea and code are from CPython implementation. +https://github.com/python/cpython/blob/b76518d43fb82ed9e5d27025d18c90a23d525c90/Lib/unittest/case.py +""" +import logging +import collections + +__all__ = ['AssertLogsMixin'] + +LoggingWatcher = collections.namedtuple('LoggingWatcher', ['records', 'output']) + + +class CapturingHandler(logging.Handler): + """ + A logging handler capturing all (raw and formatted) logging output. + """ + + def __init__(self): + super(CapturingHandler, self).__init__() + self.watcher = LoggingWatcher([], []) + + def flush(self): + pass + + def emit(self, record): + self.watcher.records.append(record) + self.watcher.output.append(self.format(record)) + + + +class AssertLogsContext(object): + """ + A context manager used to implement `TestCase.assertLogs()`. + """ + + LOGGING_FORMAT = '%(levelname)s:%(name)s:%(message)s' + + def __init__(self, test_case, logger_name, level): + self.test_case = test_case + + self.logger_name = logger_name + self.logger_states = None + self.logger = None + + if level: + # pylint: disable=protected-access,no-member + try: + name_to_level = logging._nameToLevel # type: ignore + except AttributeError: + name_to_level = logging._levelNames # type: ignore + + self.level = name_to_level.get(level, level) + else: + self.level = logging.INFO + + self.watcher = None + + def _save_logger_states(self): + self.logger_states = (self.logger.handlers[:], self.logger.level, self.logger.propagate) + + def _restore_logger_states(self): + self.logger.handlers, self.logger.level, self.logger.propagate = self.logger_states + + def __enter__(self): + if isinstance(self.logger_name, logging.Logger): + self.logger = self.logger_name + else: + self.logger = logging.getLogger(self.logger_name) + + formatter = logging.Formatter(self.LOGGING_FORMAT) + + handler = CapturingHandler() + handler.setFormatter(formatter) + + self._save_logger_states() + self.logger.handlers = [handler] + self.logger.setLevel(self.level) + self.logger.propagate = False + + self.watcher = handler.watcher + return handler.watcher + + def __exit__(self, exc_type, exc_value, tb): + self._restore_logger_states() + + if exc_type is not None: + # let unexpected exceptions pass through + return + + if not self.watcher.records: + self._raiseFailure( + "no logs of level {} or higher triggered on {}" + .format(logging.getLevelName(self.level), self.logger.name)) + + def _raiseFailure(self, message): + message = self.test_case._formatMessage(None, message) # pylint: disable=protected-access + raise self.test_case.failureException(message) + + +class AssertLogsMixin(object): + """ + A mixin that implements `TestCase.assertLogs()`. + """ + + def assertLogs(self, logger=None, level=None): + """Fail unless a log message of level *level* or higher is emitted + on *logger_name* or its children. If omitted, *level* defaults to + INFO and *logger* defaults to the root logger. + This method must be used as a context manager, and will yield + a recording object with two attributes: `output` and `records`. + At the end of the context manager, the `output` attribute will + be a list of the matching formatted log messages and the + `records` attribute will be a list of the corresponding LogRecord + objects. + Example:: + with self.assertLogs('foo', level='INFO') as cm: + logging.getLogger('foo').info('first message') + logging.getLogger('foo.bar').error('second message') + self.assertEqual(cm.output, ['INFO:foo:first message', + 'ERROR:foo.bar:second message']) + """ + return AssertLogsContext(self, logger, level) diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index 8dfd18637..4c9da84bd 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -14,9 +14,10 @@ from certbot.plugins import common from certbot.tests import util as test_util from certbot_nginx._internal import configurator from certbot_nginx._internal import nginxparser +import test_log_util -class NginxTest(test_util.ConfigTestCase): +class NginxTest(test_log_util.AssertLogsMixin, test_util.ConfigTestCase): def setUp(self): super(NginxTest, self).setUp() From 5b29e4616c4a7bae3ba18b0eca8ae245afbf97f1 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 05:30:54 +0900 Subject: [PATCH 20/72] Add simple comments --- certbot-nginx/tests/test_log_util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot-nginx/tests/test_log_util.py b/certbot-nginx/tests/test_log_util.py index f8135ca26..7aebf2151 100644 --- a/certbot-nginx/tests/test_log_util.py +++ b/certbot-nginx/tests/test_log_util.py @@ -46,8 +46,10 @@ class AssertLogsContext(object): if level: # pylint: disable=protected-access,no-member try: + # In Python 3.x name_to_level = logging._nameToLevel # type: ignore except AttributeError: + # In Python 2.7 name_to_level = logging._levelNames # type: ignore self.level = name_to_level.get(level, level) From 4ea98d830bcc3d1b980a4055243c6a6a25d8dc54 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 24 Feb 2020 12:31:16 -0800 Subject: [PATCH 21/72] remove _internal docs (#7801) --- certbot/CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 30479f25b..57fbc820b 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -13,7 +13,6 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* certbot._internal.cli is now a package split in submodules instead of a whole module. * Fix acme module warnings when response Content-Type includes params (e.g. charset). * Fixed issue where webroot plugin would incorrectly raise `Read-only file system` error when creating challenge directories (issue #7165). From 2ae090529e66a059e0e077d233f427583caa7c6c Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Tue, 25 Feb 2020 13:17:25 +0900 Subject: [PATCH 22/72] Fixed typo & some trivial documentation change --- AUTHORS.md | 2 +- certbot/CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index 8653382b8..f5b981b8e 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -266,6 +266,6 @@ Authors * [Yomna](https://github.com/ynasser) * [Yoni Jah](https://github.com/yonjah) * [YourDaddyIsHere](https://github.com/YourDaddyIsHere) +* [Yuseong Cho](https://github.com/g6123) * [Zach Shepherd](https://github.com/zjs) * [�솃訝�](https://github.com/chenxsan) -* [Yuseong Cho](https://github.com/g6123) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 837f48db2..a68c50ad0 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -17,7 +17,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* Fix nginx plguin crash when non-ASCII configuration file is being read +* Fix nginx plugin crash when non-ASCII configuration file is being read More details about these changes can be found on our GitHub repo. From ddf68aea8039b81940b7caf731405ef10613b5db Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Tue, 25 Feb 2020 13:21:10 +0900 Subject: [PATCH 23/72] Update comment in testdata file --- .../tests/testdata/etc_nginx/invalid_unicode_comments.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf b/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf index 596044cc9..4d6384535 100644 --- a/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf +++ b/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf @@ -1,5 +1,5 @@ # This configuration file is saved with EUC-KR (a.k.a. cp949) encoding, -# including some Korean alphabets. +# including some Korean letters. server { # 안녕하세요. 80번 포트에서 요청을 기다린다. From a48907920885671ef5a57bc1cda556e90c5e0e20 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Tue, 25 Feb 2020 13:26:36 +0900 Subject: [PATCH 24/72] Update parser test to better assert logging output --- certbot-nginx/tests/parser_test.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 1f9e3c996..fd5d338d3 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -496,11 +496,10 @@ class NginxParserTest(util.NginxTest): parsed = nparser._parse_files(path) # pylint: disable=protected-access self.assertEqual([], parsed) - self.assertTrue([ - True + self.assertTrue(any( + ('invalid character' in output) and ('UTF-8' in output) for output in log.output - if ('invalid character' in output) and ('UTF-8' in output) - ]) + )) def test_valid_unicode_characters_in_ssl_options(self): nparser = parser.NginxParser(self.config_path) @@ -516,11 +515,10 @@ class NginxParserTest(util.NginxTest): parsed = parser._parse_ssl_options(path) # pylint: disable=protected-access self.assertEqual([], parsed) - self.assertTrue([ - True + self.assertTrue(any( + ('invalid character' in output) and ('UTF-8' in output) for output in log.output - if ('invalid character' in output) and ('UTF-8' in output) - ]) + )) if __name__ == "__main__": unittest.main() # pragma: no cover From f4c0a9fd63c9be4cd4e745dd5f701040bcd14682 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 10:43:41 -0800 Subject: [PATCH 25/72] Split advanced pipeline (#7813) I want to do what I did in https://github.com/certbot/certbot/pull/7733 to our Azure Pipelines setup, but unfortunately this isn't currently possible. The only filters available for service hooks for the "build completed" trigger are the pipeline and build status. See ![Screen Shot 2020-02-26 at 3 04 56 PM](https://user-images.githubusercontent.com/6504915/75396464-64ad0780-58a9-11ea-97a1-3454a9754675.png) To accomplish this, I propose splitting the "advanced" pipeline into two cases. One is for builds on protected branches where we want to be notified if they fail while the other is just used to manually run tests on certain branches. --- .azure-pipelines/advanced-test.yml | 12 ++++++++++++ .azure-pipelines/advanced.yml | 10 ++-------- .azure-pipelines/release.yml | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 .azure-pipelines/advanced-test.yml diff --git a/.azure-pipelines/advanced-test.yml b/.azure-pipelines/advanced-test.yml new file mode 100644 index 000000000..b9ac9c38a --- /dev/null +++ b/.azure-pipelines/advanced-test.yml @@ -0,0 +1,12 @@ +# Advanced pipeline for running our full test suite on demand. +trigger: + # When changing these triggers, please ensure the documentation under + # "Running tests in CI" is still correct. + - azure-test-* + - test-* + +jobs: + # Any addition here should be reflected in the advanced and release pipelines. + # It is advised to declare all jobs here as templates to improve maintainability. + - template: templates/tests-suite.yml + - template: templates/installer-tests.yml diff --git a/.azure-pipelines/advanced.yml b/.azure-pipelines/advanced.yml index dda7f9bfd..7f0f5de50 100644 --- a/.azure-pipelines/advanced.yml +++ b/.azure-pipelines/advanced.yml @@ -1,12 +1,6 @@ -# Advanced pipeline for isolated checks and release purpose +# Advanced pipeline for running our full test suite on protected branches. trigger: - # When changing these triggers, please ensure the documentation under - # "Running tests in CI" is still correct. - - azure-test-* - - test-* - '*.x' -pr: - - test-* # This pipeline is also nightly run on master schedules: - cron: "0 4 * * *" @@ -17,7 +11,7 @@ schedules: always: true jobs: - # Any addition here should be reflected in the release pipeline. + # Any addition here should be reflected in the advanced-test and release pipelines. # It is advised to declare all jobs here as templates to improve maintainability. - template: templates/tests-suite.yml - template: templates/installer-tests.yml diff --git a/.azure-pipelines/release.yml b/.azure-pipelines/release.yml index aeb5ee327..e9acbc69a 100644 --- a/.azure-pipelines/release.yml +++ b/.azure-pipelines/release.yml @@ -6,7 +6,7 @@ trigger: pr: none jobs: - # Any addition here should be reflected in the advanced pipeline. + # Any addition here should be reflected in the advanced and advanced-test pipelines. # It is advised to declare all jobs here as templates to improve maintainability. - template: templates/tests-suite.yml - template: templates/installer-tests.yml From 24aa1e9127802f9c6ac459bbf91e6ff9b4595483 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 10:47:43 -0800 Subject: [PATCH 26/72] update letstest reqs (#7809) I don't fully understand why, but since I updated my macbook to macOS Catalina, the test script currently fails to run for me with the versions of our dependencies we have pinned. Updating the dependencies solves the problem though and you can see Travis also successfully running tests with these new dependencies at https://travis-ci.com/certbot/certbot/builds/150573696. --- tests/letstest/requirements.txt | 38 ++++++++++++++------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/tests/letstest/requirements.txt b/tests/letstest/requirements.txt index 64e1f6a0c..24bd77331 100644 --- a/tests/letstest/requirements.txt +++ b/tests/letstest/requirements.txt @@ -1,25 +1,19 @@ -asn1crypto==0.24.0 -awscli==1.16.157 -bcrypt==3.1.6 -boto3==1.9.146 -botocore==1.12.147 -cffi==1.12.3 -colorama==0.3.9 -cryptography==2.4.2 -docutils==0.14 -enum34==1.1.6 +bcrypt==3.1.7 +boto3==1.12.7 +botocore==1.15.7 +cffi==1.14.0 +cryptography==2.8 +docutils==0.15.2 +enum34==1.1.9 Fabric==1.14.1 -futures==3.2.0 -idna==2.8 -ipaddress==1.0.22 -jmespath==0.9.4 -paramiko==2.4.2 -pyasn1==0.4.5 +futures==3.3.0 +ipaddress==1.0.23 +jmespath==0.9.5 +paramiko==2.7.1 pycparser==2.19 PyNaCl==1.3.0 -python-dateutil==2.8.0 -PyYAML==3.10 -rsa==3.4.2 -s3transfer==0.2.0 -six==1.12.0 -urllib3==1.24.3 +python-dateutil==2.8.1 +PyYAML==5.3 +s3transfer==0.3.3 +six==1.14.0 +urllib3==1.25.8 From 8c75a9de9fe28ca0e4baf8686620a9c6b5733515 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 10:47:56 -0800 Subject: [PATCH 27/72] Remove unused notify code. (#7805) This code is unused and hasn't been modified since 2015 except for various times our files have been renamed. Let's remove it. --- certbot/certbot/_internal/notify.py | 34 ------------------- certbot/tests/notify_test.py | 52 ----------------------------- 2 files changed, 86 deletions(-) delete mode 100644 certbot/certbot/_internal/notify.py delete mode 100644 certbot/tests/notify_test.py diff --git a/certbot/certbot/_internal/notify.py b/certbot/certbot/_internal/notify.py deleted file mode 100644 index dda0a85af..000000000 --- a/certbot/certbot/_internal/notify.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Send e-mail notification to system administrators.""" - -import email -import smtplib -import socket -import subprocess - - -def notify(subject, whom, what): - """Send email notification. - - Try to notify the addressee (``whom``) by e-mail, with Subject: - defined by ``subject`` and message body by ``what``. - - """ - msg = email.message_from_string(what) - msg.add_header("From", "Certbot renewal agent ") - msg.add_header("To", whom) - msg.add_header("Subject", subject) - msg = msg.as_string() - try: - lmtp = smtplib.LMTP() - lmtp.connect() - lmtp.sendmail("root", [whom], msg) - except (smtplib.SMTPHeloError, smtplib.SMTPRecipientsRefused, - smtplib.SMTPSenderRefused, smtplib.SMTPDataError, socket.error): - # We should try using /usr/sbin/sendmail in this case - try: - proc = subprocess.Popen(["/usr/sbin/sendmail", "-t"], - stdin=subprocess.PIPE) - proc.communicate(msg) - except OSError: - return False - return True diff --git a/certbot/tests/notify_test.py b/certbot/tests/notify_test.py deleted file mode 100644 index d6f7d2239..000000000 --- a/certbot/tests/notify_test.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Tests for certbot._internal.notify.""" -import socket -import unittest - -import mock - - -class NotifyTests(unittest.TestCase): - """Tests for the notifier.""" - - @mock.patch("certbot._internal.notify.smtplib.LMTP") - def test_smtp_success(self, mock_lmtp): - from certbot._internal.notify import notify - lmtp_obj = mock.MagicMock() - mock_lmtp.return_value = lmtp_obj - self.assertTrue(notify("Goose", "auntrhody@example.com", - "The old grey goose is dead.")) - self.assertEqual(lmtp_obj.connect.call_count, 1) - self.assertEqual(lmtp_obj.sendmail.call_count, 1) - - @mock.patch("certbot._internal.notify.smtplib.LMTP") - @mock.patch("certbot._internal.notify.subprocess.Popen") - def test_smtp_failure(self, mock_popen, mock_lmtp): - from certbot._internal.notify import notify - lmtp_obj = mock.MagicMock() - mock_lmtp.return_value = lmtp_obj - lmtp_obj.sendmail.side_effect = socket.error(17) - proc = mock.MagicMock() - mock_popen.return_value = proc - self.assertTrue(notify("Goose", "auntrhody@example.com", - "The old grey goose is dead.")) - self.assertEqual(lmtp_obj.sendmail.call_count, 1) - self.assertEqual(proc.communicate.call_count, 1) - - @mock.patch("certbot._internal.notify.smtplib.LMTP") - @mock.patch("certbot._internal.notify.subprocess.Popen") - def test_everything_fails(self, mock_popen, mock_lmtp): - from certbot._internal.notify import notify - lmtp_obj = mock.MagicMock() - mock_lmtp.return_value = lmtp_obj - lmtp_obj.sendmail.side_effect = socket.error(17) - proc = mock.MagicMock() - mock_popen.return_value = proc - proc.communicate.side_effect = OSError("What we have here is a " - "failure to communicate.") - self.assertFalse(notify("Goose", "auntrhody@example.com", - "The old grey goose is dead.")) - self.assertEqual(lmtp_obj.sendmail.call_count, 1) - self.assertEqual(proc.communicate.call_count, 1) - -if __name__ == "__main__": - unittest.main() # pragma: no cover From 2f737ee292680e2f8043e0dfe3affcccc03914e8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 10:49:50 -0800 Subject: [PATCH 28/72] Change how _USE_DISTRO is set for mypy (#7804) If you run `mypy --platform darwin certbot/certbot/util.py` you'll get: ``` certbot/certbot/util.py:303: error: Name 'distro' is not defined certbot/certbot/util.py:319: error: Name 'distro' is not defined certbot/certbot/util.py:369: error: Name 'distro' is not defined ``` This is because mypy's logic for handling platform specific code is pretty simple and can't figure out what we're doing with `_USE_DISTRO` here. See https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks for more info. Setting `_USE_DISTRO` to the result of `sys.platform.startswith('linux')` solves the problem without changing the overall behavior of our code here though. This fixes part of https://github.com/certbot/certbot/issues/7803, but there's more work to be done on Windows. --- certbot/certbot/util.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index aff2952f7..e69b11543 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -25,11 +25,9 @@ from certbot._internal import lock from certbot.compat import filesystem from certbot.compat import os -if sys.platform.startswith('linux'): +_USE_DISTRO = sys.platform.startswith('linux') +if _USE_DISTRO: import distro - _USE_DISTRO = True -else: - _USE_DISTRO = False logger = logging.getLogger(__name__) From a2be8e1956c79662fd28d8b8af4802ea89cf29bf Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 10:50:20 -0800 Subject: [PATCH 29/72] Fix tests on macOS Catalina (#7794) This PR fixes the failures that can be seen at https://dev.azure.com/certbot/certbot/_build/results?buildId=1184&view=results. You can see this code running on macOS Catalina at https://dev.azure.com/certbot/certbot/_build/results?buildId=1192&view=results. --- certbot/tests/cli_test.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 3a7fb57f8..be2c8f29e 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -30,15 +30,23 @@ class TestReadFile(TempDirTestCase): # However a relative path between two different drives is invalid. So we move to # self.tempdir to ensure that we stay on the same drive. os.chdir(self.tempdir) - rel_test_path = os.path.relpath(os.path.join(self.tempdir, 'foo')) + # The read-only filesystem introduced with macOS Catalina can break + # code using relative paths below. See + # https://bugs.python.org/issue38295 for another example of this. + # Eliminating any possible symlinks in self.tempdir before passing + # it to os.path.relpath solves the problem. This is done by calling + # filesystem.realpath which removes any symlinks in the path on + # POSIX systems. + real_path = filesystem.realpath(os.path.join(self.tempdir, 'foo')) + relative_path = os.path.relpath(real_path) self.assertRaises( - argparse.ArgumentTypeError, cli.read_file, rel_test_path) + argparse.ArgumentTypeError, cli.read_file, relative_path) test_contents = b'bar\n' - with open(rel_test_path, 'wb') as f: + with open(relative_path, 'wb') as f: f.write(test_contents) - path, contents = cli.read_file(rel_test_path) + path, contents = cli.read_file(relative_path) self.assertEqual(path, os.path.abspath(path)) self.assertEqual(contents, test_contents) finally: From 6309ded92f03104f2baa9b881db9827f5fe11e4c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 14:43:28 -0800 Subject: [PATCH 30/72] Remove references to deprecated flags in Certbot. (#7509) Related to https://github.com/certbot/certbot/pull/7482, this removes some references to deprecated options in Certbot. The only references I didn't remove were: * In `certbot/tests/testdata/sample-renewal*` which contains a lot of old values and I think there's even some value in keeping them so we know if we make a change that suddenly causes old renewal configuration files to error. * In the Apache and Nginx plugins and I created https://github.com/certbot/certbot/issues/7508 to resolve that issue. --- certbot/certbot/_internal/main.py | 2 +- certbot/certbot/display/ops.py | 2 +- certbot/tests/cli_test.py | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 8674cd151..4a57dd78d 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -394,7 +394,7 @@ def _find_domains_or_certname(config, installer, question=None): :param installer: Installer object :type installer: interfaces.IInstaller - :param `str` question: Overriding dialog question to ask the user if asked + :param `str` question: Overriding default question to ask the user if asked to choose from domain names. :returns: Two-part tuple of domains and certname diff --git a/certbot/certbot/display/ops.py b/certbot/certbot/display/ops.py index eab9d251d..f24f6ed99 100644 --- a/certbot/certbot/display/ops.py +++ b/certbot/certbot/display/ops.py @@ -107,7 +107,7 @@ def choose_names(installer, question=None): :param installer: An installer object :type installer: :class:`certbot.interfaces.IInstaller` - :param `str` question: Overriding dialog question to ask the user if asked + :param `str` question: Overriding default question to ask the user if asked to choose from domain names. :returns: List of selected names diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index be2c8f29e..7d21f8bb8 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -150,7 +150,6 @@ class ParseTest(unittest.TestCase): self.assertTrue("how a certificate is deployed" in out) self.assertTrue("--webroot-path" in out) self.assertTrue("--text" not in out) - self.assertTrue("--dialog" not in out) self.assertTrue("%s" not in out) self.assertTrue("{0}" not in out) self.assertTrue("--renew-hook" not in out) @@ -211,7 +210,6 @@ class ParseTest(unittest.TestCase): self.assertTrue("how a certificate is deployed" in out) self.assertTrue("--webroot-path" in out) self.assertTrue("--text" not in out) - self.assertTrue("--dialog" not in out) self.assertTrue("%s" not in out) self.assertTrue("{0}" not in out) From fa67b7ba0fb03453fc8d03e3631d6782a54a233b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 14:44:39 -0800 Subject: [PATCH 31/72] Remove codecov (#7811) After getting a +1 from everyone on the team, this PR removes the use of `codecov` from the Certbot repo because we keep having problems with it. Two noteworthy things about this PR are: 1. I left the text at https://github.com/certbot/certbot/blob/4ea98d830bcc3d1b980a4055243c6a6a25d8dc54/.azure-pipelines/INSTALL.md#add-a-secret-variable-to-a-pipeline-like-codecov_token because I think it's useful to document how to set up a secret variable in general. 2. I'm not sure what the text "Option -e makes sure we fail fast and don't submit to codecov." in `tox.cover.py` refers to but it seems incorrect since `-e` isn't accepted or used by the script so I just deleted the line. As part of this, I said I'd open an issue to track setting up coveralls (which seems to be the only real alternative to codecov) which is at https://github.com/certbot/certbot/issues/7810. With my change, failure output looks something like: ``` $ tox -e py27-cover ... Name Stmts Miss Cover Missing ------------------------------------------------------------------------------------------ certbot/certbot/__init__.py 1 0 100% certbot/certbot/_internal/__init__.py 0 0 100% certbot/certbot/_internal/account.py 191 4 98% 62-63, 206, 337 ... certbot/tests/storage_test.py 530 0 100% certbot/tests/util_test.py 374 29 92% 211-213, 480-484, 489-499, 504-511, 545-547, 552-554 ------------------------------------------------------------------------------------------ TOTAL 14451 647 96% Command '['/path/to/certbot/dir/.tox/py27-cover/bin/python', '-m', 'coverage', 'report', '--fail-under', '100', '--include', 'certbot/*', '--show-missing']' returned non-zero exit status 2 Test coverage on certbot did not meet threshold of 100%. ERROR: InvocationError for command /Users/bmw/Development/certbot/certbot/.tox/py27-cover/bin/python tox.cover.py (exited with code 1) _________________________________________________________________________________________________________________________________________________________ summary _________________________________________________________________________________________________________________________________________________________ ERROR: py27-cover: commands failed ``` I printed the exception just so we're not throwing away information. I think it's also possible we fail for a reason other than the threshold not meeting the percentage, but I've personally never seen this, `coverage report` output is not being captured so hopefully that would inform devs if something else is going on, and saying something like "Test coverage probably did not..." seems like overkill to me personally. * remove codecov * remove unused variable group * remove codecov.yml * Improve tox.cover.py failure output. --- .azure-pipelines/templates/tests-suite.yml | 13 ------------- .codecov.yml | 18 ------------------ .travis.yml | 4 +--- certbot/README.rst | 6 +----- tools/dev_constraints.txt | 1 - tox.cover.py | 19 +++++++++++++------ 6 files changed, 15 insertions(+), 46 deletions(-) delete mode 100644 .codecov.yml diff --git a/.azure-pipelines/templates/tests-suite.yml b/.azure-pipelines/templates/tests-suite.yml index 069ea94d6..d330b7954 100644 --- a/.azure-pipelines/templates/tests-suite.yml +++ b/.azure-pipelines/templates/tests-suite.yml @@ -25,8 +25,6 @@ jobs: PYTEST_ADDOPTS: --numprocesses 4 pool: vmImage: $(IMAGE_NAME) - variables: - - group: certbot-common steps: - bash: brew install augeas condition: startswith(variables['IMAGE_NAME'], 'macOS') @@ -39,14 +37,3 @@ jobs: displayName: Install dependencies - script: python -m tox displayName: Run tox - # We do not require codecov report upload to succeed. So to avoid to break the pipeline if - # something goes wrong, each command is suffixed with a command that hides any non zero exit - # codes and echoes an informative message instead. - - bash: | - curl -s https://codecov.io/bash -o codecov-bash || echo "Failed to download codecov-bash" - chmod +x codecov-bash || echo "Failed to apply execute permissions on codecov-bash" - ./codecov-bash -F windows || echo "Codecov did not collect coverage reports" - condition: in(variables['TOXENV'], 'py37-cover', 'integration-certbot') - env: - CODECOV_TOKEN: $(codecov_token) - displayName: Publish coverage diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 0a97fffe3..000000000 --- a/.codecov.yml +++ /dev/null @@ -1,18 +0,0 @@ -coverage: - status: - project: - default: off - linux: - flags: linux - # Fixed target instead of auto set by #7173, can - # be removed when flags in Codecov are added back. - target: 97.4 - threshold: 0.1 - base: auto - windows: - flags: windows - # Fixed target instead of auto set by #7173, can - # be removed when flags in Codecov are added back. - target: 97.4 - threshold: 0.1 - base: auto diff --git a/.travis.yml b/.travis.yml index e5354898d..d498d0305 100644 --- a/.travis.yml +++ b/.travis.yml @@ -247,15 +247,13 @@ addons: # version of virtualenv. The option "-I" is set so when CERTBOT_NO_PIN is also # set, pip updates dependencies it thinks are already satisfied to avoid some # problems with its lack of real dependency resolution. -install: 'tools/pip_install.py -I codecov tox virtualenv' +install: 'tools/pip_install.py -I tox virtualenv' # Most of the time TRAVIS_RETRY is an empty string, and has no effect on the # script command. It is set only to `travis_retry` during farm tests, in # order to trigger the Travis retry feature, and compensate the inherent # flakiness of these specific tests. script: '$TRAVIS_RETRY tox' -after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux' - notifications: email: false irc: diff --git a/certbot/README.rst b/certbot/README.rst index d1b1e4fe2..5ed74f247 100644 --- a/certbot/README.rst +++ b/certbot/README.rst @@ -71,16 +71,12 @@ ACME spec: http://ietf-wg-acme.github.io/acme/ ACME working area in github: https://github.com/ietf-wg-acme/acme -|build-status| |coverage| |container| +|build-status| |container| .. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master :target: https://travis-ci.com/certbot/certbot :alt: Travis CI status -.. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg - :target: https://codecov.io/gh/certbot/certbot - :alt: Coverage status - .. |container| image:: https://quay.io/repository/letsencrypt/letsencrypt/status :target: https://quay.io/repository/letsencrypt/letsencrypt :alt: Docker Repository on Quay.io diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 7d2013c7a..cfa036435 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -18,7 +18,6 @@ boto3==1.11.7 botocore==1.14.7 cached-property==1.5.1 cloudflare==2.3.1 -codecov==2.0.15 configparser==3.7.4 contextlib2==0.6.0.post1 coverage==4.5.4 diff --git a/tox.cover.py b/tox.cover.py index 3e69a14d6..4848b2740 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function + import argparse import os import subprocess @@ -48,18 +50,23 @@ def cover(package): subprocess.check_call([sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=', pkg_dir]) - subprocess.check_call([ - sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', - '{0}/*'.format(pkg_dir), '--show-missing']) + try: + subprocess.check_call([ + sys.executable, '-m', 'coverage', 'report', '--fail-under', + str(threshold), '--include', '{0}/*'.format(pkg_dir), + '--show-missing']) + except subprocess.CalledProcessError as err: + print(err) + print('Test coverage on', pkg_dir, + 'did not meet threshold of {0}%.'.format(threshold)) + sys.exit(1) def main(): description = """ This script is used by tox.ini (and thus by Travis CI and Azure Pipelines) in order to generate separate stats for each package. It should be removed once -those packages are moved to a separate repo. - -Option -e makes sure we fail fast and don't submit to codecov.""" +those packages are moved to a separate repo.""" parser = argparse.ArgumentParser(description=description) parser.add_argument('--packages', nargs='+') From 50ea6085537dfec3bceaa4f9f4e4065de84d1407 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 15:07:33 -0800 Subject: [PATCH 32/72] Don't run advanced tests on PRs. (#7820) When I wrote https://github.com/certbot/certbot/pull/7813, I didn't understand the default behavior for pull requests if you don't specify `pr` in the yaml file. According to https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=azure-devops&tabs=yaml#pr-triggers: > If no pr triggers appear in your YAML file, pull request builds are automatically enabled for all branches... This is not the behavior we want. This PR fixes the problem by disabling builds on PRs. You should be able to see this working because the advanced tests should not run on this PR but they did run on https://github.com/certbot/certbot/pull/7811. --- .azure-pipelines/advanced-test.yml | 1 + .azure-pipelines/advanced.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.azure-pipelines/advanced-test.yml b/.azure-pipelines/advanced-test.yml index b9ac9c38a..5be29ba79 100644 --- a/.azure-pipelines/advanced-test.yml +++ b/.azure-pipelines/advanced-test.yml @@ -4,6 +4,7 @@ trigger: # "Running tests in CI" is still correct. - azure-test-* - test-* +pr: none jobs: # Any addition here should be reflected in the advanced and release pipelines. diff --git a/.azure-pipelines/advanced.yml b/.azure-pipelines/advanced.yml index 7f0f5de50..d950e6524 100644 --- a/.azure-pipelines/advanced.yml +++ b/.azure-pipelines/advanced.yml @@ -1,6 +1,7 @@ # Advanced pipeline for running our full test suite on protected branches. trigger: - '*.x' +pr: none # This pipeline is also nightly run on master schedules: - cron: "0 4 * * *" From 9f8e4507ad0cb3dbedb726dda4c46affb1eb7ad3 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Fri, 28 Feb 2020 00:44:23 +0000 Subject: [PATCH 33/72] Document safe and simple usage by services without root privileges (#7821) Certificates are public information by design: they are provided by web servers without any prior authentication required. In a public key cryptographic system, only the private key is secret information. The private key file is already created as accessible only to the root user with mode 0600, and these file permissions are set before any key content is written to the file. There is no window within which an attacker with access to the containing directory would be able to read the private key content. Older versions of Certbot (prior to 0.29.0) would create private key files with mode 0644 and rely solely on the containing directory permissions to restrict access. We therefore cannot (yet) set the relevant default directory permissions to 0755, since it is possible that a user could install Certbot, obtain a certificate, then downgrade to a pre-0.29.0 version of Certbot, then obtain another certificate. This chain of events would leave the second certificate's private key file exposed. As a compromise solution, document the fact that it is safe for the common case of non-downgrading users to change the permissions of /etc/letsencrypt/{live,archive} to 0755, and explain how to use chgrp and chmod to make the private key file readable by a non-root service user. This provides guidance on the simplest way to solve the common problem of making keys and certificates usable by services that run without root privileges, with no requirement to create a custom (and hence error-prone) executable hook. Remove the existing custom executable hook example, so that the documentation contains only the simplest and safest way to solve this very common problem. Signed-off-by: Michael Brown --- certbot/docs/using.rst | 48 ++++++++++-------------------------------- 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 27ae826bd..8ec172c24 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -485,43 +485,6 @@ If you want your hook to run only after a successful renewal, use ``certbot renew --deploy-hook /path/to/deploy-hook-script`` -For example, if you have a daemon that does not read its certificates as the -root user, a deploy hook like this can copy them to the correct location and -apply appropriate file permissions. - -/path/to/deploy-hook-script - -.. code-block:: none - - #!/bin/sh - - set -e - - for domain in $RENEWED_DOMAINS; do - case $domain in - example.com) - daemon_cert_root=/etc/some-daemon/certs - - # Make sure the certificate and private key files are - # never world readable, even just for an instant while - # we're copying them into daemon_cert_root. - umask 077 - - cp "$RENEWED_LINEAGE/fullchain.pem" "$daemon_cert_root/$domain.cert" - cp "$RENEWED_LINEAGE/privkey.pem" "$daemon_cert_root/$domain.key" - - # Apply the proper file ownership and permissions for - # the daemon to read its certificate and key. - chown some-daemon "$daemon_cert_root/$domain.cert" \ - "$daemon_cert_root/$domain.key" - chmod 400 "$daemon_cert_root/$domain.cert" \ - "$daemon_cert_root/$domain.key" - - service some-daemon restart >/dev/null - ;; - esac - done - You can also specify hooks by placing files in subdirectories of Certbot's configuration directory. Assuming your configuration directory is ``/etc/letsencrypt``, any executable files found in @@ -686,6 +649,17 @@ your (web) server configuration directly to those files (or create symlinks). During the renewal_, ``/etc/letsencrypt/live`` is updated with the latest necessary files. +For historical reasons, the containing directories are created with +permissions of ``0700`` meaning that certificates are accessible only +to servers that run as the root user. **If you will never downgrade +to an older version of Certbot**, then you can safely fix this using +``chmod 0755 /etc/letsencrypt/{live,archive}``. + +For servers that drop root privileges before attempting to read the +private key file, you will also need to use ``chgrp`` and ``chmod +0640`` to allow the server to read +``/etc/letsencrypt/live/$domain/privkey.pem``. + .. note:: ``/etc/letsencrypt/archive`` and ``/etc/letsencrypt/keys`` contain all previous keys and certificates, while ``/etc/letsencrypt/live`` symlinks to the latest versions. From 31470262110ab8e9a388d738461b08a4f489d430 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Mar 2020 11:07:15 -0800 Subject: [PATCH 34/72] Check OCSP as part of determining if the certificate is due for renewal (#7829) Fixes #1028. Doing this now because of https://community.letsencrypt.org/t/revoking-certain-certificates-on-march-4/. The new `ocsp_revoked_by_paths` function is taken from https://github.com/certbot/certbot/pull/7649 with the optional argument removed for now because it is unused. This function was added in this PR because `storage.py` uses `self.latest_common_version()` to determine which certificate should be looked at for determining renewal status at https://github.com/certbot/certbot/blob/9f8e4507ad0cb3dbedb726dda4c46affb1eb7ad3/certbot/certbot/_internal/storage.py#L939-L947 I think this is unnecessary and you can just look at the currently linked certificate, but I don't think we should be changing the logic that code has always had now. * Check OCSP status as part of determining to renew * add integration tests * add ocsp_revoked_by_paths --- .../certbot_tests/test_main.py | 17 ++++++++++ certbot/CHANGELOG.md | 2 ++ certbot/certbot/_internal/storage.py | 33 +++++++++++-------- certbot/certbot/ocsp.py | 13 +++++++- certbot/tests/storage_test.py | 33 ++++++++++++++++--- 5 files changed, 80 insertions(+), 18 deletions(-) diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 94e76cf79..f0c5edd3f 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -595,6 +595,23 @@ def test_ocsp_status_live(context): assert output.count('REVOKED') == 1, 'Expected {0} to be REVOKED'.format(cert) +def test_ocsp_renew(context): + """Test that revoked certificates are renewed.""" + # Obtain a certificate + certname = context.get_domain('ocsp-renew') + context.certbot(['--domains', certname]) + + # Test that "certbot renew" does not renew the certificate + assert_cert_count_for_lineage(context.config_dir, certname, 1) + context.certbot(['renew'], force_renew=False) + assert_cert_count_for_lineage(context.config_dir, certname, 1) + + # Revoke the certificate and test that it does renew the certificate + context.certbot(['revoke', '--cert-name', certname, '--no-delete-after-revoke']) + context.certbot(['renew'], force_renew=False) + assert_cert_count_for_lineage(context.config_dir, certname, 2) + + def test_dry_run_deactivate_authzs(context): """Test that Certbot deactivates authorizations when performing a dry run""" diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 6c1b112d7..2a934ee5b 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -13,6 +13,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed +* Certbot will now renew certificates early if they have been revoked according + to OCSP. * Fix acme module warnings when response Content-Type includes params (e.g. charset). * Fixed issue where webroot plugin would incorrectly raise `Read-only file system` error when creating challenge directories (issue #7165). diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 6a34355a8..2dac163e2 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -15,6 +15,7 @@ import certbot from certbot import crypto_util from certbot import errors from certbot import interfaces +from certbot import ocsp from certbot import util from certbot._internal import cli from certbot._internal import constants @@ -882,27 +883,33 @@ class RenewableCert(interfaces.RenewableCert): with open(target) as f: return crypto_util.get_names_from_cert(f.read()) - def ocsp_revoked(self, version=None): - # pylint: disable=unused-argument + def ocsp_revoked(self, version): """Is the specified cert version revoked according to OCSP? - Also returns True if the cert version is declared as intended - to be revoked according to Let's Encrypt OCSP extensions. - (If no version is specified, uses the current version.) - - This method is not yet implemented and currently always returns - False. + Also returns True if the cert version is declared as revoked + according to OCSP. If OCSP status could not be determined, False + is returned. :param int version: the desired version number - :returns: whether the certificate is or will be revoked + :returns: True if the certificate is revoked, otherwise, False :rtype: bool """ - # XXX: This query and its associated network service aren't - # implemented yet, so we currently return False (indicating that the - # certificate is not revoked). - return False + cert_path = self.version("cert", version) + chain_path = self.version("chain", version) + # While the RevocationChecker should return False if it failed to + # determine the OCSP status, let's ensure we don't crash Certbot by + # catching all exceptions here. + try: + return ocsp.RevocationChecker().ocsp_revoked_by_paths(cert_path, + chain_path) + except Exception as e: # pylint: disable=broad-except + logger.warning( + "An error occurred determining the OCSP status of %s.", + cert_path) + logger.debug(str(e)) + return False def autorenewal_is_enabled(self): """Is automatic renewal enabled for this cert? diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 6a95f26fa..9799c675c 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -68,8 +68,19 @@ class RevocationChecker(object): :rtype: bool """ - cert_path, chain_path = cert.cert_path, cert.chain_path + return self.ocsp_revoked_by_paths(cert.cert_path, cert.chain_path) + def ocsp_revoked_by_paths(self, cert_path, chain_path): + # type: (str, str) -> bool + """Performs the OCSP revocation check + + :param str cert_path: Certificate filepath + :param str chain_path: Certificate chain filepath + + :returns: True if revoked; False if valid or the check failed or cert is expired. + :rtype: bool + + """ if self.broken: return False diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 6208974ec..0f7620b78 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -672,10 +672,35 @@ class RenewableCertTests(BaseRenewableCertTest): errors.CertStorageError, self.test_rc._update_link_to, "elephant", 17) - def test_ocsp_revoked(self): - # XXX: This is currently hardcoded to False due to a lack of an - # OCSP server to test against. - self.assertFalse(self.test_rc.ocsp_revoked()) + @mock.patch("certbot.ocsp.RevocationChecker.ocsp_revoked_by_paths") + def test_ocsp_revoked(self, mock_checker): + # Write out test files + for kind in ALL_FOUR: + self._write_out_kind(kind, 1) + version = self.test_rc.latest_common_version() + expected_cert_path = self.test_rc.version("cert", version) + expected_chain_path = self.test_rc.version("chain", version) + + # Test with cert revoked + mock_checker.return_value = True + self.assertTrue(self.test_rc.ocsp_revoked(version)) + self.assertEqual(mock_checker.call_args[0][0], expected_cert_path) + self.assertEqual(mock_checker.call_args[0][1], expected_chain_path) + + # Test with cert not revoked + mock_checker.return_value = False + self.assertFalse(self.test_rc.ocsp_revoked(version)) + self.assertEqual(mock_checker.call_args[0][0], expected_cert_path) + self.assertEqual(mock_checker.call_args[0][1], expected_chain_path) + + # Test with error + mock_checker.side_effect = ValueError + with mock.patch("certbot._internal.storage.logger.warning") as logger: + self.assertFalse(self.test_rc.ocsp_revoked(version)) + self.assertEqual(mock_checker.call_args[0][0], expected_cert_path) + self.assertEqual(mock_checker.call_args[0][1], expected_chain_path) + log_msg = logger.call_args[0][0] + self.assertIn("An error occurred determining the OCSP status", log_msg) def test_add_time_interval(self): from certbot._internal import storage From b1fb3296e949c5ce5175321328a89a41b7dd3d12 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Mar 2020 12:36:36 -0800 Subject: [PATCH 35/72] Update changelog for 1.3.0 release --- certbot/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 2a934ee5b..493e4d5c1 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.3.0 - master +## 1.3.0 - 2020-03-03 ### Added From 6edb4e1a3924821316b9344adb9c533937426fa7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Mar 2020 12:43:02 -0800 Subject: [PATCH 36/72] Release 1.3.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 26 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- certbot/docs/cli-help.txt | 2 +- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 26 files changed, 79 insertions(+), 79 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 0e11779ba..922cae26c 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index b37ee3972..6483313b8 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index cea58e2cb..0ea3275c3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.2.0" +LE_AUTO_VERSION="1.3.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1540,18 +1540,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.2.0 \ - --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \ - --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c -acme==1.2.0 \ - --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \ - --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab -certbot-apache==1.2.0 \ - --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \ - --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3 -certbot-nginx==1.2.0 \ - --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \ - --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db +certbot==1.3.0 \ + --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ + --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 +acme==1.3.0 \ + --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ + --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d +certbot-apache==1.3.0 \ + --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ + --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 +certbot-nginx==1.3.0 \ + --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ + --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 1dbcefa75..f85311999 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.3.0.dev0' +version = '1.3.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 9376bc1c4..d5da3ed95 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 4e99ff5ff..491096d70 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 9c9d1717c..23c2903f4 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 9cde6214c..197873733 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index adaba6851..b0a016441 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index a849cef45..2c76869bc 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 51d5b8a3f..61528b3a0 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index e7e91b929..1e7829588 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index ea64f79a2..418c635b9 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index d6bedca1c..99bc1022e 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 8f5b052a2..9f5974bcb 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index fa51c2108..42c6f11bd 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index f25e348ff..64e307b52 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 8df2320ba..b18da9d97 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b180fe06a..fe707fb09 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 84ade6b08..39171bad2 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.3.0.dev0' +__version__ = '1.3.0' diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index ff49609c4..3c2289030 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -113,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.2.0 (certbot(-auto); + "". (default: CertbotACMEClient/1.3.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the user agent are: --duplicate, diff --git a/letsencrypt-auto b/letsencrypt-auto index cea58e2cb..0ea3275c3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.2.0" +LE_AUTO_VERSION="1.3.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1540,18 +1540,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.2.0 \ - --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \ - --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c -acme==1.2.0 \ - --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \ - --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab -certbot-apache==1.2.0 \ - --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \ - --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3 -certbot-nginx==1.2.0 \ - --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \ - --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db +certbot==1.3.0 \ + --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ + --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 +acme==1.3.0 \ + --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ + --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d +certbot-apache==1.3.0 \ + --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ + --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 +certbot-nginx==1.3.0 \ + --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ + --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 488d0bf2e..84473dc30 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl456ZoACgkQTRfJlc2X -dfJx8wf/addMw4kUlwu6poHqLvsifZzHAESgvq+qybgFvl5yTh2U+99PGBgxRYx+ -bENIWBi6+XB+CiVuLzIXWw/VkXh+za99orRkkVK9PI33Xr7jBMZo5Oa3JviYjl3X -PcfjioRQCD+a9Tf9RO25LXQmxn87Ql9x3nxJuk//YeSpuImFmYjIBPE4n/LPEf7z -8WHU4oxxa/bgqGCPgv6O7ZBw7ipd3g+VHcDZcNQMP4tWYb6m7x/nN61yirid7q3M -uqQ1lbitN48ISyru6xPyE6WGTvfl1SIQd21FNRETpcoesx+MTv3ApWT4dqXjZvaX -FeM55IS65e7ci6yLV9qdAbqGKzhX0Q== -=uLcV +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl5ewVUACgkQTRfJlc2X +dfJnZAf+KmxYl1YoP/FlTG5Npb64qaDdxm59SeEVJez6fZh15xq71tRPYR+4xszE +XTeyGt7uAxjYqeiBJU5xBvGC1Veprhj5AbflVOTP+5yiBr9iNWC35zmgaE63UlZ/ +V94sfL0pkax7wLngil7a0OuzUjikzK3gXOqrY8LoUdr4mAA9AhSjajWHmyY3tpDR +84GKrVhybIt0sjy/172VuPPbXZKno/clztkKMZHXNrDeL5jgJ15Va4Ts5FK0j9VT +HQvuazbGkYVCuvlp8Np5ESDje69LCJfPZxl34htoa8WNJoVIOsQWZpoXp5B5huSP +vGrh4LabZ5UDsl+k11ikHBRUpO7E5w== +=IgRH -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index e2813853b..0ea3275c3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.3.0.dev0" +LE_AUTO_VERSION="1.3.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1540,18 +1540,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.2.0 \ - --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \ - --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c -acme==1.2.0 \ - --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \ - --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab -certbot-apache==1.2.0 \ - --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \ - --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3 -certbot-nginx==1.2.0 \ - --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \ - --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db +certbot==1.3.0 \ + --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ + --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 +acme==1.3.0 \ + --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ + --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d +certbot-apache==1.3.0 \ + --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ + --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 +certbot-nginx==1.3.0 \ + --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ + --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index fefc81b37796cdecd066b9bb212d8e285fffd4d0..8c4f52d6e2992d76012ed4c5e8b65c366e53cf8c 100644 GIT binary patch literal 256 zcmV+b0ssEUOnRo|t9S`PT@Q6pdEx5)TLaw|tXK(Czj^_>8PhYd%g<>HQ?bGidlv-5 zE}9A&SzM6uK*eeB!}iF{EuInepHwl!!N=ypH>Giu0~gNuz3H_G-xK$*4GGCrIi8a} zc|vYH^EA>Vt1;uU?ETR7;K}DFE1nDBiU-xb=ODbJ#yW{#gBQmTy?oXA)L zay!@Hc;qXdB0+{NAC2*m>{YND9$B?_{$Q;DGoOCVmMhh`)kRLNXuPfjru6g4?PpB5 zE=0kjQwl0Y1m+t}(iNS8e1OH8)e$OZ#|vb^J4)NEd(wxnV%*phX+ZJIy2kBPS7%B+ G&t7>eJc8N) literal 256 zcmV+b0ssCyy6+%Byu~4aOZx+sGM?uWzM?nz3Jfz_Cb^!H(J$u3EbYx+Wmt@nUdJdC z5^ed_c;i%mwYmiD>ud)5-Lk6YWkh7j-?{0%L?KJZd%TEUG+|;|-*#FI(%jlkvk;lO zO~Yh(rw(~6mZiF)`O}!O(DYn@gPBFqESJ@M(%s)|7%F?plRgOU1Sm{BW(R9Bfx|N( zKD3y8K_?ev`Hu@xHbPDe3A34WZ3?o_F0i8u)P>>`q{3hr9CRb?E6FEIK^btkZ@Qg6 zt+z9SQGsl`GlM4_;VnLX96Xg&)Kv+HS#^V0%aEYWG9N+LKk6WSdEBew*pUM_Rk_eZ GIvjh!?|N Date: Tue, 3 Mar 2020 12:43:03 -0800 Subject: [PATCH 37/72] Add contents to certbot/CHANGELOG.md for next version --- certbot/CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 493e4d5c1..cb1f2968c 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,6 +2,22 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 1.4.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + ## 1.3.0 - 2020-03-03 ### Added From 144d4f2b446a5659e713133868eb921885ce105b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Mar 2020 12:43:04 -0800 Subject: [PATCH 38/72] Bump version to 1.4.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 922cae26c..0527b3fb5 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 6483313b8..4ec1d0a9c 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index f85311999..d6760576a 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.3.0' +version = '1.4.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d5da3ed95..67aac3231 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 491096d70..6c653967d 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 23c2903f4..c45fc8d03 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 197873733..9124f0552 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index b0a016441..2a4fd92b0 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 2c76869bc..2198fdd3e 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 61528b3a0..087766edd 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 1e7829588..1e6b96b71 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 418c635b9..4db50b56c 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 99bc1022e..49e5e3bcf 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 9f5974bcb..6c66b39dc 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 42c6f11bd..5e0900e4d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 64e307b52..98455c362 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index b18da9d97..16990a4d3 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index fe707fb09..34785f963 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 39171bad2..0ce7ff6b7 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.3.0' +__version__ = '1.4.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0ea3275c3..ca0bda2d5 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.3.0" +LE_AUTO_VERSION="1.4.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From d72a1a71d22792cecf66a8d636705b9319e8fca3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 5 Mar 2020 11:50:52 -0800 Subject: [PATCH 39/72] Fix issues with Azure Pipelines (#7838) This PR fixes two issues. First, it fixes #7814 by removing our tests on Windows Server 2012. I also added the sentence "Certbot supports Windows Server 2016 and Windows Server 2019." to https://community.letsencrypt.org/t/beta-phase-of-certbot-for-windows/105822. Second, it fixes the test failures which can be seen at https://dev.azure.com/certbot/certbot/_build/results?buildId=1309&view=results by no longer manually installing our own version of Python and instead using the one provided by Azure. These small changes are in the same PR because I wanted to fix test failures ASAP and `UsePythonVersion` is not available on Windows 2012. See https://github.com/certbot/certbot/pull/7641#discussion_r358510854. You can see tests passing with this change at https://dev.azure.com/certbot/certbot/_build/results?buildId=1311&view=results. * stop testing on win2012 * switch to UsePythonVersion --- .azure-pipelines/templates/installer-tests.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/templates/installer-tests.yml b/.azure-pipelines/templates/installer-tests.yml index 6d5672339..ea2101792 100644 --- a/.azure-pipelines/templates/installer-tests.yml +++ b/.azure-pipelines/templates/installer-tests.yml @@ -28,15 +28,13 @@ jobs: imageName: windows-2019 win2016: imageName: vs2017-win2016 - win2012r2: - imageName: vs2015-win2012r2 pool: vmImage: $(imageName) steps: - - powershell: Invoke-WebRequest https://www.python.org/ftp/python/3.8.1/python-3.8.1-amd64-webinstall.exe -OutFile C:\py3-setup.exe - displayName: Get Python - - script: C:\py3-setup.exe /quiet PrependPath=1 InstallAllUsers=1 Include_launcher=1 InstallLauncherAllUsers=1 Include_test=0 Include_doc=0 Include_dev=1 Include_debug=0 Include_tcltk=0 TargetDir=C:\py3 - displayName: Install Python + - task: UsePythonVersion@0 + inputs: + versionSpec: 3.8 + addToPath: true - task: DownloadPipelineArtifact@2 inputs: artifact: windows-installer From 7f63141e410ae7ce200c7c7408d05c87eae0c5f5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 6 Mar 2020 09:46:30 -0800 Subject: [PATCH 40/72] Add changes to the correct changelog entry (#7833) https://github.com/certbot/certbot/pull/7742 and https://github.com/certbot/certbot/pull/7738 landed after our 1.2.0 release, but the 1.2.0 changelog entry was modified instead of the one for master/1.3.0. This PR moves the changelog entries to the 1.3.0 section. --- certbot/CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index cb1f2968c..868f3c8be 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -26,6 +26,7 @@ More details about these changes can be found on our GitHub repo. determine the OCSP status of certificates. * Don't verify the existing certificate in HTTP01Response.simple_verify, for compatibility with the real-world ACME challenge checks. +* Added support for `$hostname` in nginx `server_name` directive ### Changed @@ -37,7 +38,7 @@ More details about these changes can be found on our GitHub repo. ### Fixed -* +* Fix Apache plugin to use less restrictive umask for making the challenge directory when a restrictive umask was set when certbot was started. More details about these changes can be found on our GitHub repo. @@ -46,7 +47,6 @@ More details about these changes can be found on our GitHub repo. ### Added * Added support for Cloudflare's limited-scope API Tokens -* Added support for `$hostname` in nginx `server_name` directive ### Changed @@ -59,7 +59,6 @@ More details about these changes can be found on our GitHub repo. ### Fixed * Fix collections.abc imports for Python 3.9. -* Fix Apache plugin to use less restrictive umask for making the challenge directory when a restrictive umask was set when certbot was started. More details about these changes can be found on our GitHub repo. From 69aec55ead42c9ef9231602d002825e838e9dafb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 9 Mar 2020 13:05:35 -0700 Subject: [PATCH 41/72] Remove --no-site-packages outside of certbot-auto. (#7832) --- certbot-compatibility-test/Dockerfile | 2 +- tools/_release.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index a9996f779..a6a0c93db 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -31,7 +31,7 @@ COPY certbot-nginx /opt/certbot/src/certbot-nginx/ COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ COPY tools /opt/certbot/src/tools -RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ +RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv -p python2 /opt/certbot/venv && \ /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install -U pip ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/tools/_release.sh b/tools/_release.sh index 1819adad2..97d5f5eb8 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -59,7 +59,7 @@ mv "dist.$version" "dist.$version.$(date +%s).bak" || true git tag --delete "$tag" || true tmpvenv=$(mktemp -d) -VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 $tmpvenv +VIRTUALENV_NO_DOWNLOAD=1 virtualenv -p python2 $tmpvenv . $tmpvenv/bin/activate # update setuptools/pip just like in other places in the repo pip install -U setuptools @@ -160,7 +160,7 @@ cd "dist.$version" python -m SimpleHTTPServer $PORT & # cd .. is NOT done on purpose: we make sure that all subpackages are # installed from local PyPI rather than current directory (repo root) -VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages ../venv +VIRTUALENV_NO_DOWNLOAD=1 virtualenv ../venv . ../venv/bin/activate pip install -U setuptools pip install -U pip From 78168a5248cf119289053ffcf048c4be9b2af9d6 Mon Sep 17 00:00:00 2001 From: radek-sprta Date: Wed, 11 Mar 2020 21:27:19 +0100 Subject: [PATCH 42/72] Add CloudDNS to third-party plugins (#7840) --- certbot/docs/using.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 8ec172c24..3c3ef6fad 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -280,6 +280,7 @@ pritunl_ N Y Install certificates in pritunl distributed OpenVPN proxmox_ N Y Install certificates in Proxmox Virtualization servers dns-standalone_ Y N Obtain certificates via an integrated DNS server dns-ispconfig_ Y N DNS Authentication using ISPConfig as DNS server +dns-clouddns_ Y N DNS Authentication using CloudDNS API ================== ==== ==== =============================================================== .. _haproxy: https://github.com/greenhost/certbot-haproxy @@ -291,6 +292,7 @@ dns-ispconfig_ Y N DNS Authentication using ISPConfig as DNS server .. _external-auth: https://github.com/EnigmaBridge/certbot-external-auth .. _dns-standalone: https://github.com/siilike/certbot-dns-standalone .. _dns-ispconfig: https://github.com/m42e/certbot-dns-ispconfig +.. _dns-clouddns: https://github.com/vshosting/certbot-dns-clouddns If you're interested, you can also :ref:`write your own plugin `. From 44b97df4e91a3d228bf933ee169e964070f96dd3 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 12 Mar 2020 17:29:03 +0100 Subject: [PATCH 43/72] Exposes environment variable to let hooks scripts know when the last challenge is handled (#7837) Fixes #5484 This PRs makes Certbot expose two new environment variables in the auth and cleanup hooks of the `manual` plugin: * `CERTBOT_REMAINING_CHALLENGES` contains the number of challenges that remain after the current one (so it equals to 0 when the script is called for the last challenge) * `CERTBOT_ALL_DOMAINS` contains a comma-separated list of all domains concerned by a challenge for the current certificate With these variables, an hook script can know when it is run for the last time, and then trigger appropriate finalizers for all challenges that have been executed. This will be particularly useful for certificates with a lot of domains validated with DNS-01 challenges: instead of waiting on each hook execution to check that the relevant DNS TXT entry has been inserted, these waits can be avoided thanks to the latest hook verifying all domains in one run. * Inject environment variables in manual scripts about remaining challenges * Adapt tests * Less variables and less lines * Update manual.py * Update manual_test.py * Add documentation * Add changelog --- certbot/CHANGELOG.md | 5 ++++- certbot/certbot/_internal/plugins/manual.py | 21 +++++++++++++-------- certbot/docs/using.rst | 4 +++- certbot/tests/plugins/manual_test.py | 19 +++++++++++++------ 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 868f3c8be..903af0610 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -6,7 +6,10 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added -* +* Expose two new environment variables in the authenticator and cleanup scripts used by + the `manual` plugin: `CERTBOT_REMAINING_CHALLENGES` is equal to the number of challenges + remaining after the current challenge, `CERTBOT_ALL_DOMAINS` is a comma-separated list + of all domains challenged for the current certificate. ### Changed diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index 3204fe1da..87ccdbd7e 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -35,7 +35,11 @@ class Authenticator(common.Plugin): 'is the validation string, and $CERTBOT_TOKEN is the filename of the ' 'resource requested when performing an HTTP-01 challenge. An additional ' 'cleanup script can also be provided and can use the additional variable ' - '$CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth script.') + '$CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth script.' + 'For both authenticator and cleanup script, on HTTP-01 and DNS-01 challenges,' + '$CERTBOT_REMAINING_CHALLENGES will be equal to the number of challenges that ' + 'remain after the current one, and $CERTBOT_ALL_DOMAINS contains a comma-separated ' + 'list of all domains that are challenged for the current certificate.') _DNS_INSTRUCTIONS = """\ Please deploy a DNS TXT record under the name {domain} with the following value: @@ -109,14 +113,13 @@ permitted by DNS standards.) def perform(self, achalls): # pylint: disable=missing-function-docstring self._verify_ip_logging_ok() - if self.conf('auth-hook'): - perform_achall = self._perform_achall_with_script - else: - perform_achall = self._perform_achall_manually responses = [] for achall in achalls: - perform_achall(achall) + if self.conf('auth-hook'): + self._perform_achall_with_script(achall, achalls) + else: + self._perform_achall_manually(achall) responses.append(achall.response(achall.account_key)) return responses @@ -134,9 +137,11 @@ permitted by DNS standards.) else: raise errors.PluginError('Must agree to IP logging to proceed') - def _perform_achall_with_script(self, achall): + def _perform_achall_with_script(self, achall, achalls): env = dict(CERTBOT_DOMAIN=achall.domain, - CERTBOT_VALIDATION=achall.validation(achall.account_key)) + CERTBOT_VALIDATION=achall.validation(achall.account_key), + CERTBOT_ALL_DOMAINS=','.join(one_achall.domain for one_achall in achalls), + CERTBOT_REMAINING_CHALLENGES=str(len(achalls) - achalls.index(achall) - 1)) if isinstance(achall.chall, challenges.HTTP01): env['CERTBOT_TOKEN'] = achall.chall.encode('token') else: diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 3c3ef6fad..d3c2d1582 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -738,8 +738,10 @@ the ``cleanup.sh`` script. Additionally certbot will pass relevant environment variables to these scripts: - ``CERTBOT_DOMAIN``: The domain being authenticated -- ``CERTBOT_VALIDATION``: The validation string (HTTP-01 and DNS-01 only) +- ``CERTBOT_VALIDATION``: The validation string - ``CERTBOT_TOKEN``: Resource name part of the HTTP-01 challenge (HTTP-01 only) +- ``CERTBOT_REMAINING_CHALLENGES``: Number of challenges remaining after the current challenge +- ``CERTBOT_ALL_DOMAINS``: A comma-separated list of all domains challenged for the current certificate Additionally for cleanup: diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index bd11a9538..6cdef148a 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -72,16 +72,23 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.config.manual_public_ip_logging_ok = True self.config.manual_auth_hook = ( '{0} -c "from __future__ import print_function;' - 'from certbot.compat import os; print(os.environ.get(\'CERTBOT_DOMAIN\'));' + 'from certbot.compat import os;' + 'print(os.environ.get(\'CERTBOT_DOMAIN\'));' 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' - 'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));"' + 'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));' + 'print(os.environ.get(\'CERTBOT_ALL_DOMAINS\'));' + 'print(os.environ.get(\'CERTBOT_REMAINING_CHALLENGES\'));"' .format(sys.executable)) - dns_expected = '{0}\n{1}\n{2}'.format( + dns_expected = '{0}\n{1}\n{2}\n{3}\n{4}'.format( self.dns_achall.domain, 'notoken', - self.dns_achall.validation(self.dns_achall.account_key)) - http_expected = '{0}\n{1}\n{2}'.format( + self.dns_achall.validation(self.dns_achall.account_key), + ','.join(achall.domain for achall in self.achalls), + len(self.achalls) - self.achalls.index(self.dns_achall) - 1) + http_expected = '{0}\n{1}\n{2}\n{3}\n{4}'.format( self.http_achall.domain, self.http_achall.chall.encode('token'), - self.http_achall.validation(self.http_achall.account_key)) + self.http_achall.validation(self.http_achall.account_key), + ','.join(achall.domain for achall in self.achalls), + len(self.achalls) - self.achalls.index(self.http_achall) - 1) self.assertEqual( self.auth.perform(self.achalls), From 2fd85a4f36c37cd7dfa96f129338c2b6d95dd0d8 Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Thu, 12 Mar 2020 17:37:49 +0100 Subject: [PATCH 44/72] Add serial number to certificates output (#7842) Fixes #7835 I had to mock out `get_serial_from_cert` to keep a test from failing, because `cert_path` was mocked itself in `test_report_human_readable`. Also, I kept the same style for the serial number as the recent Let's Encrypt e-mail: lowercase hexadecimal without a `0x` prefix and without colons every 2 chars. Shouldn't be a problem to change the format if required. --- certbot/CHANGELOG.md | 1 + certbot/certbot/_internal/cert_manager.py | 11 +++++++---- certbot/certbot/crypto_util.py | 14 ++++++++++++++ certbot/tests/cert_manager_test.py | 4 +++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 903af0610..2f22b5204 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -6,6 +6,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added +* Added serial number of certificate to the output of `certbot certificates` * Expose two new environment variables in the authenticator and cleanup scripts used by the `manual` plugin: `CERTBOT_REMAINING_CHALLENGES` is equal to the number of challenges remaining after the current challenge, `CERTBOT_ALL_DOMAINS` is a comma-separated list diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index e6cbd5c2c..2652b3d2c 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -276,12 +276,15 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False): status = "VALID: {0} days".format(diff.days) valid_string = "{0} ({1})".format(cert.target_expiry, status) + serial = format(crypto_util.get_serial_from_cert(cert.cert_path), 'x') certinfo.append(" Certificate Name: {0}\n" - " Domains: {1}\n" - " Expiry Date: {2}\n" - " Certificate Path: {3}\n" - " Private Key Path: {4}".format( + " Serial Number: {1}\n" + " Domains: {2}\n" + " Expiry Date: {3}\n" + " Certificate Path: {4}\n" + " Private Key Path: {5}".format( cert.lineagename, + serial, " ".join(cert.names()), valid_string, cert.fullchain, diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 9136445bc..adb972f24 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -491,3 +491,17 @@ def cert_and_chain_from_fullchain(fullchain_pem): crypto.load_certificate(crypto.FILETYPE_PEM, fullchain_pem)).decode() chain = fullchain_pem[len(cert):].lstrip() return (cert, chain) + +def get_serial_from_cert(cert_path): + """Retrieve the serial number of a certificate from certificate path + + :param str cert_path: path to a cert in PEM format + + :returns: serial number of the certificate + :rtype: int + """ + # pylint: disable=redefined-outer-name + with open(cert_path) as f: + x509 = crypto.load_certificate(crypto.FILETYPE_PEM, + f.read()) + return x509.get_serial_number() diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index eb8005b2b..bea64f09c 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -200,9 +200,11 @@ class CertificatesTest(BaseCertManagerTest): self.assertTrue(mock_utility.called) shutil.rmtree(empty_tempdir) + @mock.patch('certbot.crypto_util.get_serial_from_cert') @mock.patch('certbot._internal.cert_manager.ocsp.RevocationChecker.ocsp_revoked') - def test_report_human_readable(self, mock_revoked): + def test_report_human_readable(self, mock_revoked, mock_serial): mock_revoked.return_value = None + mock_serial.return_value = 1234567890 from certbot._internal import cert_manager import datetime import pytz From 07abe7a8d68961042ee301039dd4da87306cb1a0 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 12 Mar 2020 21:53:19 +0100 Subject: [PATCH 45/72] Reimplement tls-alpn-01 in acme (#6886) This PR is the first part of work described in #6724. It reintroduces the tls-alpn-01 challenge in `acme` module, that was introduced by #5894 and reverted by #6100. The reason it was removed in the past is because some tests showed that with `1.0.2` branch of OpenSSL, the self-signed certificate containing the authorization key is sent to the requester even if the ALPN protocol `acme-tls/1` was not declared as supported by the requester during the TLS handshake. However recent discussions lead to the conclusion that this behavior was not a security issue, because first it is coherent with the behavior with servers that do not support ALPN at all, and second it cannot make a tls-alpn-01 challenge be validated in this kind of corner case. On top of the original modifications given by #5894, I merged the code to be up-to-date with our `master`, and fixed tests to match recent evolution about not displaying the `keyAuthorization` in the deserialized JSON form of an ACME challenge. I also move the logic to verify if ALPN is available on the current system, and so that the tls-alpn-01 challenge can be used, to a dedicated static function `is_available` in `acme.challenge.TLSALPN01`. This function is used in the related tests to skip them, and will be used in the future from Certbot plugins to trigger or not the logic related to tls-alpn-01, depending on the OpenSSL version available to Python. * Reimplement TLS-ALPN-01 challenge and standalone TLS-ALPN server from #5894. * Setup a class method to check if tls-alpn-01 is supported. * Add potential missing parameter in validation for tls-alpn * Improve comments * Make a class private * Handle old versions of openssl that do not terminate the handshake when they should do. * Add changelog * Explicitly close the TLS connection by the book. * Remove unused exception * Fix lint --- acme/acme/challenges.py | 172 ++++++++++++++++++++++++--- acme/acme/crypto_util.py | 63 +++++++--- acme/acme/standalone.py | 56 ++++++++- acme/tests/challenges_test.py | 87 ++++++++++++-- acme/tests/crypto_util_test.py | 16 ++- acme/tests/standalone_test.py | 57 ++++++++- acme/tests/testdata/README | 6 +- acme/tests/testdata/rsa1024_cert.pem | 13 ++ certbot/CHANGELOG.md | 2 + 9 files changed, 425 insertions(+), 47 deletions(-) create mode 100644 acme/tests/testdata/rsa1024_cert.pem diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 39c8d6269..0b112be00 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -1,14 +1,20 @@ """ACME Identifier Validation Challenges.""" import abc +import codecs import functools import hashlib import logging +import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose import requests import six +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 +from OpenSSL import crypto +from acme import crypto_util +from acme import errors from acme import fields logger = logging.getLogger(__name__) @@ -362,29 +368,163 @@ class HTTP01(KeyAuthorizationChallenge): @ChallengeResponse.register class TLSALPN01Response(KeyAuthorizationChallengeResponse): - """ACME TLS-ALPN-01 challenge response. - - This class only allows initiating a TLS-ALPN-01 challenge returned from the - CA. Full support for responding to TLS-ALPN-01 challenges by generating and - serving the expected response certificate is not currently provided. - """ + """ACME tls-alpn-01 challenge response.""" typ = "tls-alpn-01" + PORT = 443 + """Verification port as defined by the protocol. -@Challenge.register + You can override it (e.g. for testing) by passing ``port`` to + `simple_verify`. + + """ + + ID_PE_ACME_IDENTIFIER_V1 = b"1.3.6.1.5.5.7.1.30.1" + ACME_TLS_1_PROTOCOL = "acme-tls/1" + + @property + def h(self): + """Hash value stored in challenge certificate""" + return hashlib.sha256(self.key_authorization.encode('utf-8')).digest() + + def gen_cert(self, domain, key=None, bits=2048): + """Generate tls-alpn-01 certificate. + + :param unicode domain: Domain verified by the challenge. + :param OpenSSL.crypto.PKey key: Optional private key used in + certificate generation. If not provided (``None``), then + fresh key will be generated. + :param int bits: Number of bits for newly generated key. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + if key is None: + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) + + + der_value = b"DER:" + codecs.encode(self.h, 'hex') + acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1, + critical=True, value=der_value) + + return crypto_util.gen_ss_cert(key, [domain], force_san=True, + extensions=[acme_extension]), key + + def probe_cert(self, domain, host=None, port=None): + """Probe tls-alpn-01 challenge certificate. + + :param unicode domain: domain being validated, required. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + """ + if host is None: + host = socket.gethostbyname(domain) + logger.debug('%s resolved to %s', domain, host) + if port is None: + port = self.PORT + + return crypto_util.probe_sni(host=host, port=port, name=domain, + alpn_protocols=[self.ACME_TLS_1_PROTOCOL]) + + def verify_cert(self, domain, cert): + """Verify tls-alpn-01 challenge certificate. + + :param unicode domain: Domain name being validated. + :param OpensSSL.crypto.X509 cert: Challenge certificate. + + :returns: Whether the certificate was successfully verified. + :rtype: bool + + """ + # pylint: disable=protected-access + names = crypto_util._pyopenssl_cert_or_req_all_names(cert) + logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), names) + if len(names) != 1 or names[0].lower() != domain.lower(): + return False + + for i in range(cert.get_extension_count()): + ext = cert.get_extension(i) + # FIXME: assume this is the ACME extension. Currently there is no + # way to get full OID of an unknown extension from pyopenssl. + if ext.get_short_name() == b'UNDEF': + data = ext.get_data() + return data == self.h + + return False + + # pylint: disable=too-many-arguments + def simple_verify(self, chall, domain, account_public_key, + cert=None, host=None, port=None): + """Simple verify. + + Verify ``validation`` using ``account_public_key``, optionally + probe tls-alpn-01 certificate and check using `verify_cert`. + + :param .challenges.TLSALPN01 chall: Corresponding challenge. + :param str domain: Domain name being validated. + :param JWK account_public_key: + :param OpenSSL.crypto.X509 cert: Optional certificate. If not + provided (``None``) certificate will be retrieved using + `probe_cert`. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + + :returns: ``True`` if and only if client's control of the domain has been verified. + :rtype: bool + + """ + if not self.verify(chall, account_public_key): + logger.debug("Verification of key authorization in response failed") + return False + + if cert is None: + try: + cert = self.probe_cert(domain=domain, host=host, port=port) + except errors.Error as error: + logger.debug(str(error), exc_info=True) + return False + + return self.verify_cert(domain, cert) + + +@Challenge.register # pylint: disable=too-many-ancestors class TLSALPN01(KeyAuthorizationChallenge): - """ACME tls-alpn-01 challenge. - - This class simply allows parsing the TLS-ALPN-01 challenge returned from - the CA. Full TLS-ALPN-01 support is not currently provided. - - """ - typ = "tls-alpn-01" + """ACME tls-alpn-01 challenge.""" response_cls = TLSALPN01Response + typ = response_cls.typ def validation(self, account_key, **kwargs): - """Generate validation for the challenge.""" - raise NotImplementedError() + """Generate validation. + + :param JWK account_key: + :param unicode domain: Domain verified by the challenge. + :param OpenSSL.crypto.PKey cert_key: Optional private key used + in certificate generation. If not provided (``None``), then + fresh key will be generated. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + return self.response(account_key).gen_cert( + key=kwargs.get('cert_key'), + domain=kwargs.get('domain')) + + @staticmethod + def is_supported(): + """ + Check if TLS-ALPN-01 challenge is supported on this machine. + This implies that a recent version of OpenSSL is installed (>= 1.0.2), + or a recent cryptography version shipped with the OpenSSL library is installed. + + :returns: ``True`` if TLS-ALPN-01 is supported on this machine, ``False`` otherwise. + :rtype: bool + + """ + return (hasattr(SSL.Connection, "set_alpn_protos") + and hasattr(SSL.Context, "set_alpn_select_callback")) @Challenge.register diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index dc8fedad0..f8b7e2b30 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -27,19 +27,41 @@ logger = logging.getLogger(__name__) _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore -class SSLSocket(object): +class _DefaultCertSelection(object): + def __init__(self, certs): + self.certs = certs + + def __call__(self, connection): + server_name = connection.get_servername() + return self.certs.get(server_name, None) + + +class SSLSocket(object): # pylint: disable=too-few-public-methods """SSL wrapper for sockets. :ivar socket sock: Original wrapped socket. :ivar dict certs: Mapping from domain names (`bytes`) to `OpenSSL.crypto.X509`. :ivar method: See `OpenSSL.SSL.Context` for allowed values. + :ivar alpn_selection: Hook to select negotiated ALPN protocol for + connection. + :ivar cert_selection: Hook to select certificate for connection. If given, + `certs` parameter would be ignored, and therefore must be empty. """ - def __init__(self, sock, certs, method=_DEFAULT_SSL_METHOD): + def __init__(self, sock, certs=None, + method=_DEFAULT_SSL_METHOD, alpn_selection=None, + cert_selection=None): self.sock = sock - self.certs = certs + self.alpn_selection = alpn_selection self.method = method + if not cert_selection and not certs: + raise ValueError("Neither cert_selection or certs specified.") + if cert_selection and certs: + raise ValueError("Both cert_selection and certs specified.") + if cert_selection is None: + cert_selection = _DefaultCertSelection(certs) + self.cert_selection = cert_selection def __getattr__(self, name): return getattr(self.sock, name) @@ -56,18 +78,19 @@ class SSLSocket(object): :type connection: :class:`OpenSSL.Connection` """ - server_name = connection.get_servername() - try: - key, cert = self.certs[server_name] - except KeyError: - logger.debug("Server name (%s) not recognized, dropping SSL", - server_name) + pair = self.cert_selection(connection) + if pair is None: + logger.debug("Certificate selection for server name %s failed, dropping SSL", + connection.get_servername()) return + key, cert = pair new_context = SSL.Context(self.method) new_context.set_options(SSL.OP_NO_SSLv2) new_context.set_options(SSL.OP_NO_SSLv3) new_context.use_privatekey(key) new_context.use_certificate(cert) + if self.alpn_selection is not None: + new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) class FakeConnection(object): @@ -92,6 +115,8 @@ class SSLSocket(object): context.set_options(SSL.OP_NO_SSLv2) context.set_options(SSL.OP_NO_SSLv3) context.set_tlsext_servername_callback(self._pick_certificate_cb) + if self.alpn_selection is not None: + context.set_alpn_select_callback(self.alpn_selection) ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) ssl_sock.set_accept_state() @@ -107,8 +132,9 @@ class SSLSocket(object): return ssl_sock, addr -def probe_sni(name, host, port=443, timeout=300, - method=_DEFAULT_SSL_METHOD, source_address=('', 0)): +def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-arguments + method=_DEFAULT_SSL_METHOD, source_address=('', 0), + alpn_protocols=None): """Probe SNI server for SSL certificate. :param bytes name: Byte string to send as the server name in the @@ -120,6 +146,8 @@ def probe_sni(name, host, port=443, timeout=300, :param tuple source_address: Enables multi-path probing (selection of source interface). See `socket.creation_connection` for more info. Available only in Python 2.7+. + :param alpn_protocols: Protocols to request using ALPN. + :type alpn_protocols: `list` of `bytes` :raises acme.errors.Error: In case of any problems. @@ -149,6 +177,8 @@ def probe_sni(name, host, port=443, timeout=300, client_ssl = SSL.Connection(context, client) client_ssl.set_connect_state() client_ssl.set_tlsext_host_name(name) # pyOpenSSL>=0.13 + if alpn_protocols is not None: + client_ssl.set_alpn_protos(alpn_protocols) try: client_ssl.do_handshake() client_ssl.shutdown() @@ -239,12 +269,14 @@ def _pyopenssl_cert_or_req_san(cert_or_req): def gen_ss_cert(key, domains, not_before=None, - validity=(7 * 24 * 60 * 60), force_san=True): + validity=(7 * 24 * 60 * 60), force_san=True, extensions=None): """Generate new self-signed certificate. :type domains: `list` of `unicode` :param OpenSSL.crypto.PKey key: :param bool force_san: + :param extensions: List of additional extensions to include in the cert. + :type extensions: `list` of `OpenSSL.crypto.X509Extension` If more than one domain is provided, all of the domains are put into ``subjectAltName`` X.509 extension and first domain is set as the @@ -257,10 +289,13 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16)) cert.set_version(2) - extensions = [ + if extensions is None: + extensions = [] + + extensions.append( crypto.X509Extension( b"basicConstraints", True, b"CA:TRUE, pathlen:0"), - ] + ) cert.get_subject().CN = domains[0] # TODO: what to put into cert.get_subject()? diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 236f2c234..52ac07915 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -33,7 +33,14 @@ class TLSServer(socketserver.TCPServer): def _wrap_sock(self): self.socket = crypto_util.SSLSocket( - self.socket, certs=self.certs, method=self.method) + self.socket, cert_selection=self._cert_selection, + alpn_selection=getattr(self, '_alpn_selection', None), + method=self.method) + + def _cert_selection(self, connection): # pragma: no cover + """Callback selecting certificate for connection.""" + server_name = connection.get_servername() + return self.certs.get(server_name, None) def server_bind(self): self._wrap_sock() @@ -120,6 +127,40 @@ class BaseDualNetworkedServers(object): self.threads = [] +class TLSALPN01Server(TLSServer, ACMEServerMixin): + """TLSALPN01 Server.""" + + ACME_TLS_1_PROTOCOL = b"acme-tls/1" + + def __init__(self, server_address, certs, challenge_certs, ipv6=False): + TLSServer.__init__( + self, server_address, _BaseRequestHandlerWithLogging, certs=certs, + ipv6=ipv6) + self.challenge_certs = challenge_certs + + def _cert_selection(self, connection): + # TODO: We would like to serve challenge cert only if asked for it via + # ALPN. To do this, we need to retrieve the list of protos from client + # hello, but this is currently impossible with openssl [0], and ALPN + # negotiation is done after cert selection. + # Therefore, currently we always return challenge cert, and terminate + # handshake in alpn_selection() if ALPN protos are not what we expect. + # [0] https://github.com/openssl/openssl/issues/4952 + server_name = connection.get_servername() + logger.debug("Serving challenge cert for server name %s", server_name) + return self.challenge_certs.get(server_name, None) + + def _alpn_selection(self, _connection, alpn_protos): + """Callback to select alpn protocol.""" + if len(alpn_protos) == 1 and alpn_protos[0] == self.ACME_TLS_1_PROTOCOL: + logger.debug("Agreed on %s ALPN", self.ACME_TLS_1_PROTOCOL) + return self.ACME_TLS_1_PROTOCOL + logger.debug("Cannot agree on ALPN proto. Got: %s", str(alpn_protos)) + # Explicitly close the connection now, by returning an empty string. + # See https://www.pyopenssl.org/en/stable/api/ssl.html#OpenSSL.SSL.Context.set_alpn_select_callback # pylint: disable=line-too-long + return b"" + + class HTTPServer(BaseHTTPServer.HTTPServer): """Generic HTTP Server.""" @@ -222,3 +263,16 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """ return functools.partial( cls, simple_http_resources=simple_http_resources) + + +class _BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler): + """BaseRequestHandler with logging.""" + + def log_message(self, format, *args): # pylint: disable=redefined-builtin + """Log arbitrary message.""" + logger.debug("%s - - %s", self.client_address[0], format % args) + + def handle(self): + """Handle request.""" + self.log_message("Incoming request") + socketserver.BaseRequestHandler.handle(self) diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index adebaffc5..2b44d677d 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -2,10 +2,13 @@ import unittest import josepy as jose +import OpenSSL import mock import requests from six.moves.urllib import parse as urllib_parse +from acme import errors + import test_util CERT = test_util.load_comparable_cert('cert.pem') @@ -256,30 +259,87 @@ class HTTP01Test(unittest.TestCase): class TLSALPN01ResponseTest(unittest.TestCase): def setUp(self): - from acme.challenges import TLSALPN01Response - self.msg = TLSALPN01Response(key_authorization=u'foo') + from acme.challenges import TLSALPN01 + self.chall = TLSALPN01( + token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e')) + self.domain = u'example.com' + self.domain2 = u'example2.com' + + self.response = self.chall.response(KEY) self.jmsg = { 'resource': 'challenge', 'type': 'tls-alpn-01', - 'keyAuthorization': u'foo', + 'keyAuthorization': self.response.key_authorization, } - from acme.challenges import TLSALPN01 - self.chall = TLSALPN01(token=(b'x' * 16)) - self.response = self.chall.response(KEY) - def test_to_partial_json(self): self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, - self.msg.to_partial_json()) + self.response.to_partial_json()) def test_from_json(self): from acme.challenges import TLSALPN01Response - self.assertEqual(self.msg, TLSALPN01Response.from_json(self.jmsg)) + self.assertEqual(self.response, TLSALPN01Response.from_json(self.jmsg)) def test_from_json_hashable(self): from acme.challenges import TLSALPN01Response hash(TLSALPN01Response.from_json(self.jmsg)) + def test_gen_verify_cert(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_gen_verify_cert_gen_key(self): + cert, key = self.response.gen_cert(self.domain) + self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_verify_bad_cert(self): + self.assertFalse(self.response.verify_cert(self.domain, + test_util.load_cert('cert.pem'))) + + def test_verify_bad_domain(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertFalse(self.response.verify_cert(self.domain2, cert)) + + def test_simple_verify_bad_key_authorization(self): + key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) + self.response.simple_verify(self.chall, "local", key2.public_key()) + + @mock.patch('acme.challenges.TLSALPN01Response.verify_cert', autospec=True) + def test_simple_verify(self, mock_verify_cert): + mock_verify_cert.return_value = mock.sentinel.verification + self.assertEqual( + mock.sentinel.verification, self.response.simple_verify( + self.chall, self.domain, KEY.public_key(), + cert=mock.sentinel.cert)) + mock_verify_cert.assert_called_once_with( + self.response, self.domain, mock.sentinel.cert) + + @mock.patch('acme.challenges.socket.gethostbyname') + @mock.patch('acme.challenges.crypto_util.probe_sni') + def test_probe_cert(self, mock_probe_sni, mock_gethostbyname): + mock_gethostbyname.return_value = '127.0.0.1' + self.response.probe_cert('foo.com') + mock_gethostbyname.assert_called_once_with('foo.com') + mock_probe_sni.assert_called_once_with( + host='127.0.0.1', port=self.response.PORT, name='foo.com', + alpn_protocols=['acme-tls/1']) + + self.response.probe_cert('foo.com', host='8.8.8.8') + mock_probe_sni.assert_called_with( + host='8.8.8.8', port=mock.ANY, name='foo.com', + alpn_protocols=['acme-tls/1']) + + @mock.patch('acme.challenges.TLSALPN01Response.probe_cert') + def test_simple_verify_false_on_probe_error(self, mock_probe_cert): + mock_probe_cert.side_effect = errors.Error + self.assertFalse(self.response.simple_verify( + self.chall, self.domain, KEY.public_key())) + class TLSALPN01Test(unittest.TestCase): @@ -309,8 +369,13 @@ class TLSALPN01Test(unittest.TestCase): self.assertRaises( jose.DeserializationError, TLSALPN01.from_json, self.jmsg) - def test_validation(self): - self.assertRaises(NotImplementedError, self.msg.validation, KEY) + @mock.patch('acme.challenges.TLSALPN01Response.gen_cert') + def test_validation(self, mock_gen_cert): + mock_gen_cert.return_value = ('cert', 'key') + self.assertEqual(('cert', 'key'), self.msg.validation( + KEY, cert_key=mock.sentinel.cert_key, domain=mock.sentinel.domain)) + mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key, + domain=mock.sentinel.domain) class DNSTest(unittest.TestCase): diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index 41640ed60..ff08a5405 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -18,7 +18,6 @@ import test_util class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" - def setUp(self): self.cert = test_util.load_comparable_cert('rsa2048_cert.pem') key = test_util.load_pyopenssl_private_key('rsa2048_key.pem') @@ -32,7 +31,8 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init def server_bind(self): # pylint: disable=missing-docstring - self.socket = SSLSocket(socket.socket(), certs=certs) + self.socket = SSLSocket(socket.socket(), + certs) socketserver.TCPServer.server_bind(self) self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) @@ -73,6 +73,18 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): socket.setdefaulttimeout(original_timeout) +class SSLSocketTest(unittest.TestCase): + """Tests for acme.crypto_util.SSLSocket.""" + + def test_ssl_socket_invalid_arguments(self): + from acme.crypto_util import SSLSocket + with self.assertRaises(ValueError): + _ = SSLSocket(None, {'sni': ('key', 'cert')}, + cert_selection=lambda _: None) + with self.assertRaises(ValueError): + _ = SSLSocket(None) + + class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 83ced12b0..e2817b29c 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -10,7 +10,10 @@ from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import challenges +from acme import crypto_util +from acme import errors from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module + import test_util @@ -84,6 +87,59 @@ class HTTP01ServerTest(unittest.TestCase): self.assertFalse(self._test_http01(add=False)) +@unittest.skipIf(not challenges.TLSALPN01.is_supported(), "pyOpenSSL too old") +class TLSALPN01ServerTest(unittest.TestCase): + """Test for acme.standalone.TLSALPN01Server.""" + + def setUp(self): + self.certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa2048_key.pem'), + test_util.load_cert('rsa2048_cert.pem'), + )} + # Use different certificate for challenge. + self.challenge_certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa1024_key.pem'), + test_util.load_cert('rsa1024_cert.pem'), + )} + from acme.standalone import TLSALPN01Server + self.server = TLSALPN01Server(("localhost", 0), certs=self.certs, + challenge_certs=self.challenge_certs) + # pylint: disable=no-member + self.thread = threading.Thread(target=self.server.serve_forever) + self.thread.start() + + def tearDown(self): + self.server.shutdown() # pylint: disable=no-member + self.thread.join() + + # TODO: This is not implemented yet, see comments in standalone.py + # def test_certs(self): + # host, port = self.server.socket.getsockname()[:2] + # cert = crypto_util.probe_sni( + # b'localhost', host=host, port=port, timeout=1) + # # Expect normal cert when connecting without ALPN. + # self.assertEqual(jose.ComparableX509(cert), + # jose.ComparableX509(self.certs[b'localhost'][1])) + + def test_challenge_certs(self): + host, port = self.server.socket.getsockname()[:2] + cert = crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"acme-tls/1"]) + # Expect challenge cert when connecting with ALPN. + self.assertEqual( + jose.ComparableX509(cert), + jose.ComparableX509(self.challenge_certs[b'localhost'][1]) + ) + + def test_bad_alpn(self): + host, port = self.server.socket.getsockname()[:2] + with self.assertRaises(errors.Error): + crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"bad-alpn"]) + + class BaseDualNetworkedServersTest(unittest.TestCase): """Test for acme.standalone.BaseDualNetworkedServers.""" @@ -138,7 +194,6 @@ class BaseDualNetworkedServersTest(unittest.TestCase): class HTTP01DualNetworkedServersTest(unittest.TestCase): """Tests for acme.standalone.HTTP01DualNetworkedServers.""" - def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) diff --git a/acme/tests/testdata/README b/acme/tests/testdata/README index dfe3f5405..d65cc3018 100644 --- a/acme/tests/testdata/README +++ b/acme/tests/testdata/README @@ -10,6 +10,8 @@ and for the CSR: openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -outform DER > csr.der -and for the certificate: +and for the certificates: - openssl req -key rsa2047_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 > rsa2048_cert.pem + openssl req -key rsa1024_key.pem -new -subj '/CN=example.com' -x509 > rsa1024_cert.pem diff --git a/acme/tests/testdata/rsa1024_cert.pem b/acme/tests/testdata/rsa1024_cert.pem new file mode 100644 index 000000000..1b7912181 --- /dev/null +++ b/acme/tests/testdata/rsa1024_cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB/TCCAWagAwIBAgIJAOyRIBs3QT8QMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV +BAMMC2V4YW1wbGUuY29tMB4XDTE4MDQyMzEwMzE0NFoXDTE4MDUyMzEwMzE0NFow +FjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ +AoGBAJqJ87R8aVwByONxgQA9hwgvQd/QqI1r1UInXhEF2VnEtZGtUWLi100IpIqr +Mq4qusDwNZ3g8cUPtSkvJGs89djoajMDIJP7lQUEKUYnYrI0q755Tr/DgLWSk7iW +l5ezym0VzWUD0/xXUz8yRbNMTjTac80rS5SZk2ja2wWkYlRJAgMBAAGjUzBRMB0G +A1UdDgQWBBSsaX0IVZ4XXwdeffVAbG7gnxSYjTAfBgNVHSMEGDAWgBSsaX0IVZ4X +XwdeffVAbG7gnxSYjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB +ADe7SVmvGH2nkwVfONk8TauRUDkePN1CJZKFb2zW1uO9ANJ2v5Arm/OQp0BG/xnI +Djw/aLTNVESF89oe15dkrUErtcaF413MC1Ld5lTCaJLHLGqDKY69e02YwRuxW7jY +qarpt7k7aR5FbcfO5r4V/FK/Gvp4Dmoky8uap7SJIW6x +-----END CERTIFICATE----- diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 2f22b5204..36547fdd1 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -11,6 +11,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). the `manual` plugin: `CERTBOT_REMAINING_CHALLENGES` is equal to the number of challenges remaining after the current challenge, `CERTBOT_ALL_DOMAINS` is a comma-separated list of all domains challenged for the current certificate. +* Added TLS-ALPN-01 challenge support in the `acme` library. Support of this + challenge in the Certbot client is planned to be added in a future release. ### Changed From 809cb516c918575bc1688141dfe9b4da001d6570 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 13 Mar 2020 17:56:35 +0100 Subject: [PATCH 46/72] Fix acme compliance to RFC 8555 (#7176) This PR is an alternative to #7125. Instead of disabling the strict mode on Pebble, this PR fixes the JWS payloads regarding RFC 8555 to be compliant, and allow certbot to work with Pebble v2.1.0+. * Fix acme compliance to RFC 8555. * Working mixin * Activate back pebble strict mode * Use mixin for type * Update dependencies * Fix also in fields_to_partial_json * Update pebble * Add changelog --- acme/acme/challenges.py | 3 +- acme/acme/client.py | 3 + acme/acme/messages.py | 13 ++-- acme/acme/mixins.py | 65 +++++++++++++++++++ acme/tests/challenges_test.py | 13 ++++ acme/tests/client_test.py | 3 +- acme/tests/messages_test.py | 14 ++++ .../utils/acme_server.py | 2 +- .../utils/pebble_artifacts.py | 2 +- certbot-nginx/local-oldest-requirements.txt | 4 +- certbot-nginx/setup.py | 4 +- certbot/CHANGELOG.md | 3 +- certbot/local-oldest-requirements.txt | 2 +- certbot/setup.py | 2 +- 14 files changed, 116 insertions(+), 17 deletions(-) create mode 100644 acme/acme/mixins.py diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 0b112be00..b9c6b7eb2 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -16,6 +16,7 @@ from OpenSSL import crypto from acme import crypto_util from acme import errors from acme import fields +from acme.mixins import ResourceMixin, TypeMixin logger = logging.getLogger(__name__) @@ -34,7 +35,7 @@ class Challenge(jose.TypedJSONObjectWithFields): return UnrecognizedChallenge.from_json(jobj) -class ChallengeResponse(jose.TypedJSONObjectWithFields): +class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge response.""" TYPES = {} # type: dict diff --git a/acme/acme/client.py b/acme/acme/client.py index cecb727c7..cbe543f91 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -25,6 +25,7 @@ from acme.magic_typing import Dict from acme.magic_typing import List from acme.magic_typing import Set from acme.magic_typing import Text +from acme.mixins import VersionedLEACMEMixin logger = logging.getLogger(__name__) @@ -987,6 +988,8 @@ class ClientNetwork(object): :rtype: `josepy.JWS` """ + if isinstance(obj, VersionedLEACMEMixin): + obj.le_acme_version = acme_version jobj = obj.json_dumps(indent=2).encode() if obj else b'' logger.debug('JWS payload:\n%s', jobj) kwargs = { diff --git a/acme/acme/messages.py b/acme/acme/messages.py index f8f4bfbe7..90059a6fb 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -9,6 +9,7 @@ from acme import errors from acme import fields from acme import jws from acme import util +from acme.mixins import ResourceMixin try: from collections.abc import Hashable @@ -356,13 +357,13 @@ class Registration(ResourceBody): @Directory.register -class NewRegistration(Registration): +class NewRegistration(ResourceMixin, Registration): """New registration.""" resource_type = 'new-reg' resource = fields.Resource(resource_type) -class UpdateRegistration(Registration): +class UpdateRegistration(ResourceMixin, Registration): """Update registration.""" resource_type = 'reg' resource = fields.Resource(resource_type) @@ -498,13 +499,13 @@ class Authorization(ResourceBody): @Directory.register -class NewAuthorization(Authorization): +class NewAuthorization(ResourceMixin, Authorization): """New authorization.""" resource_type = 'new-authz' resource = fields.Resource(resource_type) -class UpdateAuthorization(Authorization): +class UpdateAuthorization(ResourceMixin, Authorization): """Update authorization.""" resource_type = 'authz' resource = fields.Resource(resource_type) @@ -522,7 +523,7 @@ class AuthorizationResource(ResourceWithURI): @Directory.register -class CertificateRequest(jose.JSONObjectWithFields): +class CertificateRequest(ResourceMixin, jose.JSONObjectWithFields): """ACME new-cert request. :ivar josepy.util.ComparableX509 csr: @@ -548,7 +549,7 @@ class CertificateResource(ResourceWithURI): @Directory.register -class Revocation(jose.JSONObjectWithFields): +class Revocation(ResourceMixin, jose.JSONObjectWithFields): """Revocation message. :ivar .ComparableX509 certificate: `OpenSSL.crypto.X509` wrapped in diff --git a/acme/acme/mixins.py b/acme/acme/mixins.py new file mode 100644 index 000000000..1cd050ccc --- /dev/null +++ b/acme/acme/mixins.py @@ -0,0 +1,65 @@ +"""Useful mixins for Challenge and Resource objects""" + + +class VersionedLEACMEMixin(object): + """This mixin stores the version of Let's Encrypt's endpoint being used.""" + @property + def le_acme_version(self): + """Define the version of ACME protocol to use""" + return getattr(self, '_le_acme_version', 1) + + @le_acme_version.setter + def le_acme_version(self, version): + # We need to use object.__setattr__ to not depend on the specific implementation of + # __setattr__ in current class (eg. jose.TypedJSONObjectWithFields raises AttributeError + # for any attempt to set an attribute to make objects immutable). + object.__setattr__(self, '_le_acme_version', version) + + def __setattr__(self, key, value): + if key == 'le_acme_version': + # Required for @property to operate properly. See comment above. + object.__setattr__(self, key, value) + else: + super(VersionedLEACMEMixin, self).__setattr__(key, value) # pragma: no cover + + +class ResourceMixin(VersionedLEACMEMixin): + """ + This mixin generates a RFC8555 compliant JWS payload + by removing the `resource` field if needed (eg. ACME v2 protocol). + """ + def to_partial_json(self): + """See josepy.JSONDeserializable.to_partial_json()""" + return _safe_jobj_compliance(super(ResourceMixin, self), + 'to_partial_json', 'resource') + + def fields_to_partial_json(self): + """See josepy.JSONObjectWithFields.fields_to_partial_json()""" + return _safe_jobj_compliance(super(ResourceMixin, self), + 'fields_to_partial_json', 'resource') + + +class TypeMixin(VersionedLEACMEMixin): + """ + This mixin allows generation of a RFC8555 compliant JWS payload + by removing the `type` field if needed (eg. ACME v2 protocol). + """ + def to_partial_json(self): + """See josepy.JSONDeserializable.to_partial_json()""" + return _safe_jobj_compliance(super(TypeMixin, self), + 'to_partial_json', 'type') + + def fields_to_partial_json(self): + """See josepy.JSONObjectWithFields.fields_to_partial_json()""" + return _safe_jobj_compliance(super(TypeMixin, self), + 'fields_to_partial_json', 'type') + + +def _safe_jobj_compliance(instance, jobj_method, uncompliant_field): + if hasattr(instance, jobj_method): + jobj = getattr(instance, jobj_method)() + if instance.le_acme_version == 2: + jobj.pop(uncompliant_field, None) + return jobj + + raise AttributeError('Method {0}() is not implemented.'.format(jobj_method)) # pragma: no cover diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index 2b44d677d..433c7bb82 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -478,5 +478,18 @@ class DNSResponseTest(unittest.TestCase): self.msg.check_validation(self.chall, KEY.public_key())) +class JWSPayloadRFC8555Compliant(unittest.TestCase): + """Test for RFC8555 compliance of JWS generated from resources/challenges""" + def test_challenge_payload(self): + from acme.challenges import HTTP01Response + + challenge_body = HTTP01Response() + challenge_body.le_acme_version = 2 + + jobj = challenge_body.json_dumps(indent=2).encode() + # RFC8555 states that challenge responses must have an empty payload. + self.assertEqual(jobj, b'{}') + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index a4966140f..1e132d79f 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -16,6 +16,7 @@ from acme import errors from acme import jws as acme_jws from acme import messages from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module +from acme.mixins import VersionedLEACMEMixin import messages_test import test_util @@ -886,7 +887,7 @@ class ClientV2Test(ClientTestBase): self.client.net.get.assert_not_called() -class MockJSONDeSerializable(jose.JSONDeSerializable): +class MockJSONDeSerializable(VersionedLEACMEMixin, jose.JSONDeSerializable): # pylint: disable=missing-docstring def __init__(self, value): self.value = value diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index b9b70266b..d53fb764c 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -453,6 +453,7 @@ class OrderResourceTest(unittest.TestCase): 'authorizations': None, }) + class NewOrderTest(unittest.TestCase): """Tests for acme.messages.NewOrder.""" @@ -467,5 +468,18 @@ class NewOrderTest(unittest.TestCase): }) +class JWSPayloadRFC8555Compliant(unittest.TestCase): + """Test for RFC8555 compliance of JWS generated from resources/challenges""" + def test_message_payload(self): + from acme.messages import NewAuthorization + + new_order = NewAuthorization() + new_order.le_acme_version = 2 + + jobj = new_order.json_dumps(indent=2).encode() + # RFC8555 states that JWS bodies must not have a resource field. + self.assertEqual(jobj, b'{}') + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 5483251e6..b14bd4a32 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -131,7 +131,7 @@ class ACMEServer(object): environ['PEBBLE_AUTHZREUSE'] = '100' self._launch_process( - [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053'], + [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053', '-strict'], env=environ) self._launch_process( diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index 2b1557928..7fe03b990 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -7,7 +7,7 @@ import requests from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT -PEBBLE_VERSION = 'v2.2.1' +PEBBLE_VERSION = 'v2.3.0' ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index cee142934..1782f15ba 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,3 +1,3 @@ # Remember to update setup.py to match the package versions below. -acme[dev]==1.0.0 -certbot[dev]==1.1.0 +-e acme[dev] +-e certbot[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 34785f963..0d62e7d55 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -9,8 +9,8 @@ version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=1.0.0', - 'certbot>=1.1.0', + 'acme>=1.4.0.dev0', + 'certbot>=1.4.0.dev0', 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 36547fdd1..0bbc9b7ff 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -20,7 +20,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* +* When using an RFC 8555 compliant endpoint, the `acme` library no longer sends the + `resource` field in any requests or the `type` field when responding to challenges. More details about these changes can be found on our GitHub repo. diff --git a/certbot/local-oldest-requirements.txt b/certbot/local-oldest-requirements.txt index f6d158890..0acc68652 100644 --- a/certbot/local-oldest-requirements.txt +++ b/certbot/local-oldest-requirements.txt @@ -1,2 +1,2 @@ # Remember to update setup.py to match the package versions below. -acme[dev]==0.40.0 +-e acme[dev] diff --git a/certbot/setup.py b/certbot/setup.py index d19327e5e..514aec8c5 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -36,7 +36,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=0.40.0', + 'acme>=1.4.0.dev0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. From 9a256ca4fee624a5aa1b901de939f0dc6dd4e641 Mon Sep 17 00:00:00 2001 From: Blake Bourque Date: Fri, 13 Mar 2020 15:26:15 -0400 Subject: [PATCH 47/72] Fix plugin links --- certbot/docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 25d832761..06d0e1b8d 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -247,8 +247,8 @@ built-in Standalone authenticator, implement just one interface. There are also `~certbot.interfaces.IDisplay` plugins, which can change how prompts are displayed to a user. -.. _interfaces.py: https://github.com/certbot/certbot/blob/master/certbot/interfaces.py -.. _plugins/common.py: https://github.com/certbot/certbot/blob/master/certbot/plugins/common.py#L34 +.. _interfaces.py: https://github.com/certbot/certbot/blob/master/certbot/certbot/interfaces.py +.. _plugins/common.py: https://github.com/certbot/certbot/blob/master/certbot/certbot/plugins/common.py#L45 Authenticators From 06599a1e18cd809dc4db26990da67fcdfc3316d4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 16 Mar 2020 09:43:48 -0700 Subject: [PATCH 48/72] Cleanup more pylint issues (#7848) This PR builds on #7657 and cleans up additional unnecessary pylint comments and some stray comments referring to pylint: disable comments that have been deleted that I didn't notice in my review of that PR. * Remove stray pylint link. * Cleanup more pylint comments * Cleanup magic_typing imports * Remove unneeded pylint: enable comments --- acme/acme/magic_typing.py | 1 - acme/tests/client_test.py | 1 - acme/tests/crypto_util_test.py | 1 - acme/tests/magic_typing_test.py | 4 ++-- acme/tests/messages_test.py | 1 - acme/tests/standalone_test.py | 1 - certbot-apache/certbot_apache/_internal/augeasparser.py | 2 +- certbot-apache/certbot_apache/_internal/entrypoint.py | 2 -- certbot-apache/certbot_apache/_internal/interfaces.py | 1 - certbot-apache/tests/augeasnode_test.py | 1 - certbot-apache/tests/http_01_test.py | 1 - certbot-nginx/certbot_nginx/_internal/configurator.py | 1 - certbot-nginx/tests/parser_test.py | 1 - certbot/certbot/_internal/cli/helpful.py | 4 +--- certbot/certbot/_internal/plugins/disco.py | 3 --- certbot/certbot/compat/filesystem.py | 3 --- certbot/tests/compat/filesystem_test.py | 2 -- certbot/tests/display/completer_test.py | 1 - certbot/tests/error_handler_test.py | 3 --- certbot/tests/hook_test.py | 1 - certbot/tests/log_test.py | 1 - certbot/tests/main_test.py | 1 - certbot/tests/plugins/disco_test.py | 1 - certbot/tests/plugins/selection_test.py | 1 - certbot/tests/plugins/standalone_test.py | 3 --- 25 files changed, 4 insertions(+), 38 deletions(-) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index d6b1ff056..7c5231c75 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -11,6 +11,5 @@ try: # mypy doesn't respect modifying sys.modules from typing import * # pylint: disable=wildcard-import, unused-wildcard-import from typing import Collection, IO # type: ignore - # pylint: enable=unused-import except ImportError: sys.modules[__name__] = TypingClass() diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 1e132d79f..010974a32 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -15,7 +15,6 @@ from acme import challenges from acme import errors from acme import jws as acme_jws from acme import messages -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from acme.mixins import VersionedLEACMEMixin import messages_test import test_util diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index ff08a5405..705a3c856 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -11,7 +11,6 @@ import six from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import errors -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module import test_util diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 23dfe3367..60b4a5df4 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -18,7 +18,7 @@ class MagicTypingTest(unittest.TestCase): sys.modules['typing'] = typing_class_mock if 'acme.magic_typing' in sys.modules: del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text # pylint: disable=no-name-in-module + from acme.magic_typing import Text self.assertEqual(Text, text_mock) del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing @@ -31,7 +31,7 @@ class MagicTypingTest(unittest.TestCase): sys.modules['typing'] = None if 'acme.magic_typing' in sys.modules: del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text # pylint: disable=no-name-in-module + from acme.magic_typing import Text self.assertTrue(Text is None) del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index d53fb764c..d36e2cc99 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -5,7 +5,6 @@ import josepy as jose import mock from acme import challenges -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module import test_util CERT = test_util.load_comparable_cert('cert.der') diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index e2817b29c..8c08ab89b 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -12,7 +12,6 @@ from six.moves import socketserver # type: ignore # pylint: disable=import-err from acme import challenges from acme import crypto_util from acme import errors -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module import test_util diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index e1d7c941d..f85d80923 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -64,7 +64,7 @@ Translates over to: "/files/etc/apache2/apache2.conf/bLoCk[1]", ] """ -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Set from certbot import errors from certbot.compat import os diff --git a/certbot-apache/certbot_apache/_internal/entrypoint.py b/certbot-apache/certbot_apache/_internal/entrypoint.py index e31e1f4eb..79337b381 100644 --- a/certbot-apache/certbot_apache/_internal/entrypoint.py +++ b/certbot-apache/certbot_apache/_internal/entrypoint.py @@ -1,6 +1,4 @@ """ Entry point for Apache Plugin """ -# Pylint does not like disutils.version when running inside a venv. -# See: https://github.com/PyCQA/pylint/issues/73 from distutils.version import LooseVersion from certbot import util diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 1b67be5c8..647790c41 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -102,7 +102,6 @@ For this reason the internal representation of data should not ignore the case. import abc import six -from acme.magic_typing import Any, Dict, Optional, Tuple # pylint: disable=unused-import, no-name-in-module @six.add_metaclass(abc.ABCMeta) diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 9d663a05f..8417bc283 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -3,7 +3,6 @@ import mock import util -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot_apache._internal import assertions diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 422a76443..85b17ca28 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -5,7 +5,6 @@ import errno import mock from acme import challenges -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot.compat import filesystem diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 459950aa1..ddab48512 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -1,5 +1,4 @@ """Nginx Configuration""" -# https://github.com/PyCQA/pylint/issues/73 from distutils.version import LooseVersion import logging import re diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index fd5d338d3..72cfc0716 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -4,7 +4,6 @@ import re import shutil import unittest -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot.compat import os from certbot_nginx._internal import nginxparser diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index e63ab4b87..31d9396e5 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -11,9 +11,7 @@ import zope.interface from zope.interface import interfaces as zope_interfaces -# pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Any, Dict, Optional -# pylint: enable=unused-import, no-name-in-module +from acme.magic_typing import Any, Dict from certbot import crypto_util from certbot import errors diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index d98a4cb0c..f1d89f06a 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -192,9 +192,6 @@ class PluginsRegistry(Mapping): # This prevents deadlock caused by plugins acquiring a lock # and ensures at least one concurrent Certbot instance will run # successfully. - - # Pylint checks for super init, but also claims the super - # has no __init__member self._plugins = collections.OrderedDict(sorted(six.iteritems(plugins))) @classmethod diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index 88c2916fa..b9b6e5cc6 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -6,8 +6,6 @@ import os # pylint: disable=os-module-forbidden import stat from acme.magic_typing import List -from acme.magic_typing import Tuple # pylint: disable=unused-import -from acme.magic_typing import Union # pylint: disable=unused-import try: import ntsecuritycon @@ -17,7 +15,6 @@ try: import win32file import pywintypes import winerror - # pylint: enable=import-error except ImportError: POSIX_MODE = True else: diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index e721bbd48..fdfb1ffe9 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -13,11 +13,9 @@ import certbot.tests.util as test_util from certbot.tests.util import TempDirTestCase try: - # pylint: disable=import-error import win32api import win32security import ntsecuritycon - # pylint: enable=import-error POSIX_MODE = False except ImportError: POSIX_MODE = True diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 5ddf69266..a183fd14f 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -10,7 +10,6 @@ import unittest import mock from six.moves import reload_module # pylint: disable=import-error -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.compat import filesystem # pylint: disable=ungrouped-imports from certbot.compat import os # pylint: disable=ungrouped-imports import certbot.tests.util as test_util # pylint: disable=ungrouped-imports diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 45fec7f39..011313208 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -6,9 +6,6 @@ import unittest import mock -from acme.magic_typing import Callable # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module from certbot.compat import os diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index a3bba57d2..54d0fcf67 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -3,7 +3,6 @@ import unittest import mock -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import util from certbot.compat import filesystem diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 3b9adbbf2..5b0918ce5 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -9,7 +9,6 @@ import mock import six from acme import messages -from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import util from certbot._internal import constants diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 7ebe5e66a..8b2645876 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -18,7 +18,6 @@ import pytz import six from six.moves import reload_module # pylint: disable=import-error -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import crypto_util from certbot import errors from certbot import interfaces # pylint: disable=unused-import diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index 6d3c7d97e..eec0795e3 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -8,7 +8,6 @@ import pkg_resources import six import zope.interface -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot._internal.plugins import standalone diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index ac846af7b..c66473ad1 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -5,7 +5,6 @@ import unittest import mock import zope.component -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot._internal.plugins.disco import PluginsRegistry diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 5d9ff5244..701abe109 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -11,9 +11,6 @@ import six from acme import challenges from acme import standalone as acme_standalone # pylint: disable=unused-import -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot.tests import acme_util From 9e3c348dff75e6db4c27ae3e343d9e151d50862e Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 23 Mar 2020 16:49:52 -0700 Subject: [PATCH 49/72] Disable TLS session tickets in Apache (#7771) Fixes #7350. This PR changes the parsed modules from a `set` to a `dict`, with the filepath argument as the value. Accordingly, after calling `enable_mod` to enable `ssl_module`, modules now need to be re-parsed, so call `reset_modules`. * Add mechanism for selecting apache config file, based on work done in #7191. * Check OpenSSL version * Remove os imports * debian override still needs os * Reformat remaining apache tests with modules dict syntax * Clean up more apache tests * Switch from property to method for openssl and add tests for coverage. * Sometimes the dict location will be None in which case we should in fact return None * warn thoroughly and consistently in openssl_version function * update tests for new warnings * read file as bytes, and factor out the open for testing * normalize ssl_module_location path to account for being relative to server root * Use byte literals in a python 2 and 3 compatible way * string does need to be a literal * patch builtins open * add debug, remove space * Add test to check if OpenSSL detection is working on different systems * fix relative test location for cwd * put on its own line in test case * Revert test file to status in master. * Call augeas load before reparsing modules to pick up the changes * fix grep, tail, and mod_ssl location on centos * strip the trailing whitespace from fedora * just use LooseVersion in test * call apache2ctl on debian systems * Use sudo for apache2ctl command * add check to make sure we're getting a version * Add boolean so we don't warn on debian/ubuntu before trying to enable mod_ssl * Reduce warnings while testing by setting mock _openssl_version. * Make sure we're not throwing away any unwritten changes to the config * test last warning case for coverage * text changes for clarity --- certbot-apache/MANIFEST.in | 2 +- .../certbot_apache/_internal/apache_util.py | 13 ++ .../certbot_apache/_internal/configurator.py | 96 ++++++++- .../certbot_apache/_internal/constants.py | 1 + .../certbot_apache/_internal/override_arch.py | 4 - .../_internal/override_centos.py | 4 - .../_internal/override_darwin.py | 4 - .../_internal/override_debian.py | 3 - .../_internal/override_fedora.py | 5 - .../_internal/override_gentoo.py | 4 - .../certbot_apache/_internal/override_suse.py | 4 - .../certbot_apache/_internal/parser.py | 47 +++-- .../current-options-ssl-apache.conf | 19 ++ .../old-options-ssl-apache.conf} | 0 certbot-apache/tests/autohsts_test.py | 12 +- certbot-apache/tests/centos_test.py | 2 +- certbot-apache/tests/configurator_test.py | 193 +++++++++++++----- certbot-apache/tests/debian_test.py | 20 +- certbot-apache/tests/fedora_test.py | 2 +- certbot-apache/tests/gentoo_test.py | 2 +- certbot-apache/tests/http_01_test.py | 16 +- certbot-apache/tests/parser_test.py | 8 +- certbot-apache/tests/util.py | 6 +- certbot/CHANGELOG.md | 7 +- tests/letstest/scripts/test_apache2.sh | 16 ++ .../letstest/scripts/test_openssl_version.py | 30 +++ 26 files changed, 373 insertions(+), 147 deletions(-) create mode 100644 certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf rename certbot-apache/certbot_apache/_internal/{options-ssl-apache.conf => tls_configs/old-options-ssl-apache.conf} (100%) create mode 100644 tests/letstest/scripts/test_openssl_version.py diff --git a/certbot-apache/MANIFEST.in b/certbot-apache/MANIFEST.in index 2316983bb..9e4913a03 100644 --- a/certbot-apache/MANIFEST.in +++ b/certbot-apache/MANIFEST.in @@ -1,7 +1,7 @@ include LICENSE.txt include README.rst recursive-include tests * -include certbot_apache/_internal/options-ssl-apache.conf recursive-include certbot_apache/_internal/augeas_lens *.aug +recursive-include certbot_apache/_internal/tls_configs *.conf global-exclude __pycache__ global-exclude *.py[cod] diff --git a/certbot-apache/certbot_apache/_internal/apache_util.py b/certbot-apache/certbot_apache/_internal/apache_util.py index 085ccddc8..ebc8a26c9 100644 --- a/certbot-apache/certbot_apache/_internal/apache_util.py +++ b/certbot-apache/certbot_apache/_internal/apache_util.py @@ -5,6 +5,8 @@ import logging import re import subprocess +import pkg_resources + from certbot import errors from certbot import util @@ -241,3 +243,14 @@ def _get_runtime_cfg(command): "loaded because Apache is misconfigured.") return stdout + +def find_ssl_apache_conf(prefix): + """ + Find a TLS Apache config file in the dedicated storage. + :param str prefix: prefix of the TLS Apache config file to find + :return: the path the TLS Apache config file + :rtype: str + """ + return pkg_resources.resource_filename( + "certbot_apache", + os.path.join("_internal", "tls_configs", "{0}-options-ssl-apache.conf".format(prefix))) diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 8daa28173..fc386404d 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -1,6 +1,7 @@ """Apache Configurator.""" # pylint: disable=too-many-lines from collections import defaultdict +from distutils.version import LooseVersion import copy import fnmatch import logging @@ -8,7 +9,6 @@ import re import socket import time -import pkg_resources import six import zope.component import zope.interface @@ -110,14 +110,29 @@ class ApacheConfigurator(common.Installer): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def option(self, key): """Get a value from options""" return self.options.get(key) + def pick_apache_config(self, warn_on_no_mod_ssl=True): + """ + Pick the appropriate TLS Apache configuration file for current version of Apache and OS. + + :param bool warn_on_no_mod_ssl: True if we should warn if mod_ssl is not found. + + :return: the path to the TLS Apache configuration file to use + :rtype: str + """ + # Disabling TLS session tickets is supported by Apache 2.4.11+ and OpenSSL 1.0.2l+. + # So for old versions of Apache we pick a configuration without this option. + openssl_version = self.openssl_version(warn_on_no_mod_ssl) + if self.version < (2, 4, 11) or not openssl_version or\ + LooseVersion(openssl_version) < LooseVersion('1.0.2l'): + return apache_util.find_ssl_apache_conf("old") + return apache_util.find_ssl_apache_conf("current") + def _prepare_options(self): """ Set the values possibly changed by command line parameters to @@ -184,6 +199,7 @@ class ApacheConfigurator(common.Installer): """ version = kwargs.pop("version", None) use_parsernode = kwargs.pop("use_parsernode", False) + openssl_version = kwargs.pop("openssl_version", None) super(ApacheConfigurator, self).__init__(*args, **kwargs) # Add name_server association dict @@ -209,6 +225,7 @@ class ApacheConfigurator(common.Installer): self.parser = None self.parser_root = None self.version = version + self._openssl_version = openssl_version self.vhosts = None self.options = copy.deepcopy(self.OS_DEFAULTS) self._enhance_func = {"redirect": self._enable_redirect, @@ -225,6 +242,52 @@ class ApacheConfigurator(common.Installer): """Full absolute path to digest of updated SSL configuration file.""" return os.path.join(self.config.config_dir, constants.UPDATED_MOD_SSL_CONF_DIGEST) + def _open_module_file(self, ssl_module_location): + """Extract the open lines of openssl_version for testing purposes""" + try: + with open(ssl_module_location, mode="rb") as f: + contents = f.read() + except IOError as error: + logger.debug(str(error), exc_info=True) + return None + return contents + + def openssl_version(self, warn_on_no_mod_ssl=True): + """Lazily retrieve openssl version + + :param bool warn_on_no_mod_ssl: `True` if we should warn if mod_ssl is not found. Set to + `False` when we know we'll try to enable mod_ssl later. This is currently debian/ubuntu, + when called from `prepare`. + + :return: the OpenSSL version as a string, or None. + :rtype: str or None + """ + if self._openssl_version: + return self._openssl_version + # Step 1. Check for LoadModule directive + try: + ssl_module_location = self.parser.modules['ssl_module'] + except KeyError: + if warn_on_no_mod_ssl: + logger.warning("Could not find ssl_module; not disabling session tickets.") + return None + if not ssl_module_location: + logger.warning("Could not find ssl_module; not disabling session tickets.") + return None + ssl_module_location = self.parser.standard_path_from_server_root(ssl_module_location) + # Step 2. Grep in the .so for openssl version + contents = self._open_module_file(ssl_module_location) + if not contents: + logger.warning("Unable to read ssl_module file; not disabling session tickets.") + return None + # looks like: OpenSSL 1.0.2s 28 May 2019 + matches = re.findall(br"OpenSSL ([0-9]\.[^ ]+) ", contents) + if not matches: + logger.warning("Could not find OpenSSL version; not disabling session tickets.") + return None + self._openssl_version = matches[0].decode('UTF-8') + return self._openssl_version + def prepare(self): """Prepare the authenticator/installer. @@ -271,8 +334,12 @@ class ApacheConfigurator(common.Installer): # Get all of the available vhosts self.vhosts = self.get_virtual_hosts() + # We may try to enable mod_ssl later. If so, we shouldn't warn if we can't find it now. + # This is currently only true for debian/ubuntu. + warn_on_no_mod_ssl = not self.option("handle_modules") self.install_ssl_options_conf(self.mod_ssl_conf, - self.updated_mod_ssl_conf_digest) + self.updated_mod_ssl_conf_digest, + warn_on_no_mod_ssl) # Prevent two Apache plugins from modifying a config at once try: @@ -1241,6 +1308,14 @@ class ApacheConfigurator(common.Installer): self.enable_mod("socache_shmcb", temp=temp) if "ssl_module" not in self.parser.modules: self.enable_mod("ssl", temp=temp) + # Make sure we're not throwing away any unwritten changes to the config + self.parser.ensure_augeas_state() + self.parser.aug.load() + self.parser.reset_modules() # Reset to load the new ssl_module path + # Call again because now we can gate on openssl version + self.install_ssl_options_conf(self.mod_ssl_conf, + self.updated_mod_ssl_conf_digest, + warn_on_no_mod_ssl=True) def make_vhost_ssl(self, nonssl_vhost): """Makes an ssl_vhost version of a nonssl_vhost. @@ -2454,14 +2529,19 @@ class ApacheConfigurator(common.Installer): self.restart() self.parser.reset_modules() - def install_ssl_options_conf(self, options_ssl, options_ssl_digest): - """Copy Certbot's SSL options file into the system's config dir if required.""" + def install_ssl_options_conf(self, options_ssl, options_ssl_digest, warn_on_no_mod_ssl=True): + """Copy Certbot's SSL options file into the system's config dir if required. + + :param bool warn_on_no_mod_ssl: True if we should warn if mod_ssl is not found. + """ # XXX if we ever try to enforce a local privilege boundary (eg, running # certbot for unprivileged users via setuid), this function will need # to be modified. - return common.install_version_controlled_file(options_ssl, options_ssl_digest, - self.option("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) + apache_config_path = self.pick_apache_config(warn_on_no_mod_ssl) + + return common.install_version_controlled_file( + options_ssl, options_ssl_digest, apache_config_path, constants.ALL_SSL_OPTIONS_HASHES) def enable_autohsts(self, _unused_lineage, domains): """ diff --git a/certbot-apache/certbot_apache/_internal/constants.py b/certbot-apache/certbot_apache/_internal/constants.py index a37bebac5..68d29406e 100644 --- a/certbot-apache/certbot_apache/_internal/constants.py +++ b/certbot-apache/certbot_apache/_internal/constants.py @@ -26,6 +26,7 @@ ALL_SSL_OPTIONS_HASHES = [ '06675349e457eae856120cdebb564efe546f0b87399f2264baeb41e442c724c7', '5cc003edd93fb9cd03d40c7686495f8f058f485f75b5e764b789245a386e6daf', '007cd497a56a3bb8b6a2c1aeb4997789e7e38992f74e44cc5d13a625a738ac73', + '34783b9e2210f5c4a23bced2dfd7ec289834716673354ed7c7abf69fe30192a3', ] """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC""" diff --git a/certbot-apache/certbot_apache/_internal/override_arch.py b/certbot-apache/certbot_apache/_internal/override_arch.py index 2765bd238..54202c087 100644 --- a/certbot-apache/certbot_apache/_internal/override_arch.py +++ b/certbot-apache/certbot_apache/_internal/override_arch.py @@ -1,9 +1,7 @@ """ Distribution specific override class for Arch Linux """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import configurator @@ -26,6 +24,4 @@ class ArchConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 2ab160c2f..0de882519 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -1,14 +1,12 @@ """ Distribution specific override class for CentOS family (RHEL, Fedora) """ import logging -import pkg_resources import zope.interface from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import os from certbot.errors import MisconfigurationError from certbot_apache._internal import apache_util from certbot_apache._internal import configurator @@ -37,8 +35,6 @@ class CentOSConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def config_test(self): diff --git a/certbot-apache/certbot_apache/_internal/override_darwin.py b/certbot-apache/certbot_apache/_internal/override_darwin.py index 00faff623..f19823866 100644 --- a/certbot-apache/certbot_apache/_internal/override_darwin.py +++ b/certbot-apache/certbot_apache/_internal/override_darwin.py @@ -1,9 +1,7 @@ """ Distribution specific override class for macOS """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import configurator @@ -26,6 +24,4 @@ class DarwinConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/other", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) diff --git a/certbot-apache/certbot_apache/_internal/override_debian.py b/certbot-apache/certbot_apache/_internal/override_debian.py index 77ced6a3f..c977fb43e 100644 --- a/certbot-apache/certbot_apache/_internal/override_debian.py +++ b/certbot-apache/certbot_apache/_internal/override_debian.py @@ -1,7 +1,6 @@ """ Distribution specific override class for Debian family (Ubuntu/Debian) """ import logging -import pkg_resources import zope.interface from certbot import errors @@ -34,8 +33,6 @@ class DebianConfigurator(configurator.ApacheConfigurator): handle_modules=True, handle_sites=True, challenge_location="/etc/apache2", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def enable_site(self, vhost): diff --git a/certbot-apache/certbot_apache/_internal/override_fedora.py b/certbot-apache/certbot_apache/_internal/override_fedora.py index 8197b0dcd..2436c76cc 100644 --- a/certbot-apache/certbot_apache/_internal/override_fedora.py +++ b/certbot-apache/certbot_apache/_internal/override_fedora.py @@ -1,11 +1,9 @@ """ Distribution specific override class for Fedora 29+ """ -import pkg_resources import zope.interface from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser @@ -31,9 +29,6 @@ class FedoraConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - # TODO: eventually newest version of Fedora will need their own config - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def config_test(self): diff --git a/certbot-apache/certbot_apache/_internal/override_gentoo.py b/certbot-apache/certbot_apache/_internal/override_gentoo.py index c215771e6..6b7416c0d 100644 --- a/certbot-apache/certbot_apache/_internal/override_gentoo.py +++ b/certbot-apache/certbot_apache/_internal/override_gentoo.py @@ -1,9 +1,7 @@ """ Distribution specific override class for Gentoo Linux """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser @@ -29,8 +27,6 @@ class GentooConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def _prepare_options(self): diff --git a/certbot-apache/certbot_apache/_internal/override_suse.py b/certbot-apache/certbot_apache/_internal/override_suse.py index 0c9219e6d..895e0cb05 100644 --- a/certbot-apache/certbot_apache/_internal/override_suse.py +++ b/certbot-apache/certbot_apache/_internal/override_suse.py @@ -1,9 +1,7 @@ """ Distribution specific override class for OpenSUSE """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import configurator @@ -26,6 +24,4 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 992672913..82d7df6aa 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -9,7 +9,6 @@ import six from acme.magic_typing import Dict from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util @@ -52,7 +51,7 @@ class ApacheParser(object): "version 1.2.0 or higher, please make sure you have you have " "those installed.") - self.modules = set() # type: Set[str] + self.modules = {} # type: Dict[str, str] self.parser_paths = {} # type: Dict[str, List[str]] self.variables = {} # type: Dict[str, str] @@ -249,14 +248,14 @@ class ApacheParser(object): def add_mod(self, mod_name): """Shortcut for updating parser modules.""" if mod_name + "_module" not in self.modules: - self.modules.add(mod_name + "_module") + self.modules[mod_name + "_module"] = None if "mod_" + mod_name + ".c" not in self.modules: - self.modules.add("mod_" + mod_name + ".c") + self.modules["mod_" + mod_name + ".c"] = None def reset_modules(self): """Reset the loaded modules list. This is called from cleanup to clear temporarily loaded modules.""" - self.modules = set() + self.modules = {} self.update_modules() self.parse_modules() @@ -267,7 +266,7 @@ class ApacheParser(object): the iteration issue. Else... parse and enable mods at same time. """ - mods = set() # type: Set[str] + mods = {} # type: Dict[str, str] matches = self.find_dir("LoadModule") iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration @@ -281,8 +280,8 @@ class ApacheParser(object): mod_name = self.get_arg(match_name) mod_filename = self.get_arg(match_filename) if mod_name and mod_filename: - mods.add(mod_name) - mods.add(os.path.basename(mod_filename)[:-2] + "c") + mods[mod_name] = mod_filename + mods[os.path.basename(mod_filename)[:-2] + "c"] = mod_filename else: logger.debug("Could not read LoadModule directive from Augeas path: %s", match_name[6:]) @@ -621,7 +620,7 @@ class ApacheParser(object): def exclude_dirs(self, matches): """Exclude directives that are not loaded into the configuration.""" - filters = [("ifmodule", self.modules), ("ifdefine", self.variables)] + filters = [("ifmodule", self.modules.keys()), ("ifdefine", self.variables)] valid_matches = [] @@ -662,6 +661,25 @@ class ApacheParser(object): return True + def standard_path_from_server_root(self, arg): + """Ensure paths are consistent and absolute + + :param str arg: Argument of directive + + :returns: Standardized argument path + :rtype: str + """ + # Remove beginning and ending quotes + arg = arg.strip("'\"") + + # Standardize the include argument based on server root + if not arg.startswith("/"): + # Normpath will condense ../ + arg = os.path.normpath(os.path.join(self.root, arg)) + else: + arg = os.path.normpath(arg) + return arg + def _get_include_path(self, arg): """Converts an Apache Include directive into Augeas path. @@ -682,16 +700,7 @@ class ApacheParser(object): # if matchObj.group() != arg: # logger.error("Error: Invalid regexp characters in %s", arg) # return [] - - # Remove beginning and ending quotes - arg = arg.strip("'\"") - - # Standardize the include argument based on server root - if not arg.startswith("/"): - # Normpath will condense ../ - arg = os.path.normpath(os.path.join(self.root, arg)) - else: - arg = os.path.normpath(arg) + arg = self.standard_path_from_server_root(arg) # Attempts to add a transform to the file if one does not already exist if os.path.isdir(arg): diff --git a/certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf b/certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf new file mode 100644 index 000000000..32a2c3335 --- /dev/null +++ b/certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf @@ -0,0 +1,19 @@ +# This file contains important security parameters. If you modify this file +# manually, Certbot will be unable to automatically provide future security +# updates. Instead, Certbot will print and log an error message with a path to +# the up-to-date file that you will need to refer to when manually updating +# this file. + +SSLEngine on + +# Intermediate configuration, tweak to your needs +SSLProtocol all -SSLv2 -SSLv3 -TLSv1 -TLSv1.1 +SSLCipherSuite ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 +SSLHonorCipherOrder off +SSLSessionTickets off + +SSLOptions +StrictRequire + +# Add vhost name to log entries: +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined +LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common diff --git a/certbot-apache/certbot_apache/_internal/options-ssl-apache.conf b/certbot-apache/certbot_apache/_internal/tls_configs/old-options-ssl-apache.conf similarity index 100% rename from certbot-apache/certbot_apache/_internal/options-ssl-apache.conf rename to certbot-apache/certbot_apache/_internal/tls_configs/old-options-ssl-apache.conf diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index c9901ecdb..8e4f15d6b 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -20,10 +20,10 @@ class AutoHSTSTest(util.ApacheTest): self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) - self.config.parser.modules.add("headers_module") - self.config.parser.modules.add("mod_headers.c") - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["headers_module"] = None + self.config.parser.modules["mod_headers.c"] = None + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") @@ -42,8 +42,8 @@ class AutoHSTSTest(util.ApacheTest): @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.restart") @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod") def test_autohsts_enable_headers_mod(self, mock_enable, _restart): - self.config.parser.modules.discard("headers_module") - self.config.parser.modules.discard("mod_header.c") + self.config.parser.modules.pop("headers_module", None) + self.config.parser.modules.pop("mod_header.c", None) self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) self.assertTrue(mock_enable.called) diff --git a/certbot-apache/tests/centos_test.py b/certbot-apache/tests/centos_test.py index 55fee3faa..5e334c51e 100644 --- a/certbot-apache/tests/centos_test.py +++ b/certbot-apache/tests/centos_test.py @@ -126,7 +126,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest): return mod_val return "" mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() + self.config.parser.modules = {} self.config.parser.variables = {} with mock.patch("certbot.util.get_os_info") as mock_osi: diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index cbb052155..54f4c9251 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -341,9 +341,9 @@ class MultipleVhostsTest(util.ApacheTest): def test_deploy_cert_enable_new_vhost(self): # Create ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.assertFalse(ssl_vhost.enabled) self.config.deploy_cert( @@ -377,9 +377,9 @@ class MultipleVhostsTest(util.ApacheTest): # pragma: no cover def test_deploy_cert(self): - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None # Patch _add_dummy_ssl_directives to make sure we write them correctly # pylint: disable=protected-access orig_add_dummy = self.config._add_dummy_ssl_directives @@ -459,9 +459,9 @@ class MultipleVhostsTest(util.ApacheTest): method is called with an invalid vhost parameter. Currently this tests that a PluginError is appropriately raised when important directives are missing in an SSL module.""" - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None def side_effect(*args): """Mocks case where an SSLCertificateFile directive can be found @@ -544,7 +544,8 @@ class MultipleVhostsTest(util.ApacheTest): call_found = True self.assertTrue(call_found) - def test_prepare_server_https(self): + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https(self, mock_reset): mock_enable = mock.Mock() self.config.enable_mod = mock_enable @@ -570,7 +571,8 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(mock_add_dir.call_count, 2) - def test_prepare_server_https_named_listen(self): + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https_named_listen(self, mock_reset): mock_find = mock.Mock() mock_find.return_value = ["test1", "test2", "test3"] mock_get = mock.Mock() @@ -608,7 +610,8 @@ class MultipleVhostsTest(util.ApacheTest): # self.config.prepare_server_https("8080", temp=True) # self.assertEqual(self.listens, 0) - def test_prepare_server_https_needed_listen(self): + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https_needed_listen(self, mock_reset): mock_find = mock.Mock() mock_find.return_value = ["test1", "test2"] mock_get = mock.Mock() @@ -624,8 +627,8 @@ class MultipleVhostsTest(util.ApacheTest): self.config.prepare_server_https("443") self.assertEqual(mock_add_dir.call_count, 1) - def test_prepare_server_https_mixed_listen(self): - + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https_mixed_listen(self, mock_reset): mock_find = mock.Mock() mock_find.return_value = ["test1", "test2"] mock_get = mock.Mock() @@ -904,7 +907,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache._internal.display_ops.select_vhost") @mock.patch("certbot.util.exe_exists") def test_enhance_unknown_vhost(self, mock_exe, mock_sel_vhost, mock_get): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None mock_exe.return_value = True ssl_vh1 = obj.VirtualHost( "fp1", "ap1", set([obj.Addr(("*", "443"))]), @@ -942,8 +945,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling(self, mock_exe): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True @@ -969,8 +972,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling_twice(self, mock_exe): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True @@ -997,8 +1000,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_ocsp_unsupported_apache_version(self, mock_exe): mock_exe.return_value = True self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.config.get_version = mock.Mock(return_value=(2, 2, 0)) self.config.choose_vhost("certbot.demo") @@ -1021,8 +1024,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_http_header_hsts(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None mock_exe.return_value = True # This will create an ssl vhost for certbot.demo @@ -1042,9 +1045,9 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(len(hsts_header), 4) def test_http_header_hsts_twice(self): - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None # skip the enable mod - self.config.parser.modules.add("headers_module") + self.config.parser.modules["headers_module"] = None # This will create an ssl vhost for encryption-example.demo self.config.choose_vhost("encryption-example.demo") @@ -1060,8 +1063,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_http_header_uir(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None mock_exe.return_value = True @@ -1084,9 +1087,9 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(len(uir_header), 4) def test_http_header_uir_twice(self): - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None # skip the enable mod - self.config.parser.modules.add("headers_module") + self.config.parser.modules["headers_module"] = None # This will create an ssl vhost for encryption-example.demo self.config.choose_vhost("encryption-example.demo") @@ -1101,7 +1104,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_well_formed_http(self, mock_exe, _): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2)) @@ -1127,7 +1130,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_rewrite_rule_exists(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.parser.add_dir( self.vh_truth[3].path, "RewriteRule", ["Unknown"]) @@ -1136,7 +1139,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_rewrite_engine_exists(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.parser.add_dir( self.vh_truth[3].path, "RewriteEngine", "on") @@ -1146,7 +1149,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_with_existing_rewrite(self, mock_exe, _): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2, 0)) @@ -1180,7 +1183,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_with_old_https_redirection(self, mock_exe, _): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2, 0)) @@ -1209,7 +1212,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_with_conflict(self): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None ssl_vh = obj.VirtualHost( "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]), @@ -1222,7 +1225,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_two_domains_one_vhost(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) # Creates ssl vhost for the domain @@ -1237,7 +1240,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_from_previous_run(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.choose_vhost("red.blue.purple.com") self.config.enhance("red.blue.purple.com", "redirect") @@ -1250,7 +1253,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enhance, "green.blue.purple.com", "redirect") def test_create_own_redirect(self): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) # For full testing... give names... self.vh_truth[1].name = "default.com" @@ -1261,7 +1264,7 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(len(self.config.vhosts), 13) def test_create_own_redirect_for_old_apache_version(self): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 2)) # For full testing... give names... self.vh_truth[1].name = "default.com" @@ -1326,9 +1329,9 @@ class MultipleVhostsTest(util.ApacheTest): def test_deploy_cert_not_parsed_path(self): # Make sure that we add include to root config for vhosts when # handle-sites is false - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None tmp_path = filesystem.realpath(tempfile.mkdtemp("vhostroot")) filesystem.chmod(tmp_path, 0o755) mock_p = "certbot_apache._internal.configurator.ApacheConfigurator._get_ssl_vhost_path" @@ -1441,8 +1444,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator._choose_vhosts_wildcard") def test_enhance_wildcard_after_install(self, mock_choose): # pylint: disable=protected-access - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None self.vh_truth[3].ssl = True self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] self.config.enhance("*.certbot.demo", "ensure-http-header", @@ -1453,8 +1456,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_enhance_wildcard_no_install(self, mock_choose): self.vh_truth[3].ssl = True mock_choose.return_value = [self.vh_truth[3]] - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None self.config.enhance("*.certbot.demo", "ensure-http-header", "Upgrade-Insecure-Requests") self.assertTrue(mock_choose.called) @@ -1638,7 +1641,7 @@ class MultiVhostsTest(util.ApacheTest): @certbot_util.patch_get_utility() def test_make_vhost_ssl_with_existing_rewrite_rule(self, mock_get_utility): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[4]) @@ -1658,7 +1661,7 @@ class MultiVhostsTest(util.ApacheTest): @certbot_util.patch_get_utility() def test_make_vhost_ssl_with_existing_rewrite_conds(self, mock_get_utility): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[3]) @@ -1700,7 +1703,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.config.updated_mod_ssl_conf_digest) def _current_ssl_options_hash(self): - return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC")) + return crypto_util.sha256sum(self.config.pick_apache_config()) def _assert_current_file(self): self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) @@ -1736,7 +1739,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.assertFalse(mock_logger.warning.called) self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) self.assertEqual(crypto_util.sha256sum( - self.config.option("MOD_SSL_CONF_SRC")), + self.config.pick_apache_config()), self._current_ssl_options_hash()) self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf), self._current_ssl_options_hash()) @@ -1752,19 +1755,99 @@ class InstallSslOptionsConfTest(util.ApacheTest): "%s has been manually modified; updated file " "saved to %s. We recommend updating %s for security purposes.") self.assertEqual(crypto_util.sha256sum( - self.config.option("MOD_SSL_CONF_SRC")), + self.config.pick_apache_config()), self._current_ssl_options_hash()) # only print warning once with mock.patch("certbot.plugins.common.logger") as mock_logger: self._call() self.assertFalse(mock_logger.warning.called) - def test_current_file_hash_in_all_hashes(self): + def test_ssl_config_files_hash_in_all_hashes(self): + """ + It is really critical that all TLS Apache config files have their SHA256 hash registered in + constants.ALL_SSL_OPTIONS_HASHES. Otherwise Certbot will mistakenly assume that the config + file has been manually edited by the user, and will refuse to update it. + This test ensures that all necessary hashes are present. + """ from certbot_apache._internal.constants import ALL_SSL_OPTIONS_HASHES - self.assertTrue(self._current_ssl_options_hash() in ALL_SSL_OPTIONS_HASHES, - "Constants.ALL_SSL_OPTIONS_HASHES must be appended" - " with the sha256 hash of self.config.mod_ssl_conf when it is updated.") + import pkg_resources + tls_configs_dir = pkg_resources.resource_filename( + "certbot_apache", os.path.join("_internal", "tls_configs")) + all_files = [os.path.join(tls_configs_dir, name) for name in os.listdir(tls_configs_dir) + if name.endswith('options-ssl-apache.conf')] + self.assertTrue(all_files) + for one_file in all_files: + file_hash = crypto_util.sha256sum(one_file) + self.assertTrue(file_hash in ALL_SSL_OPTIONS_HASHES, + "Constants.ALL_SSL_OPTIONS_HASHES must be appended with the sha256 " + "hash of {0} when it is updated.".format(one_file)) + + def test_openssl_version(self): + self.config._openssl_version = None + some_string_contents = b""" + SSLOpenSSLConfCmd + OpenSSL configuration command + SSLv3 not supported by this version of OpenSSL + '%s': invalid OpenSSL configuration command + OpenSSL 1.0.2g 1 Mar 2016 + OpenSSL + AH02407: "SSLOpenSSLConfCmd %s %s" failed for %s + AH02556: "SSLOpenSSLConfCmd %s %s" applied to %s + OpenSSL 1.0.2g 1 Mar 2016 + """ + self.config.parser.modules['ssl_module'] = '/fake/path' + with mock.patch("certbot_apache._internal.configurator." + "ApacheConfigurator._open_module_file") as mock_omf: + mock_omf.return_value = some_string_contents + self.assertEqual(self.config.openssl_version(), "1.0.2g") + + def test_current_version(self): + self.config.version = (2, 4, 10) + self.config._openssl_version = '1.0.2m' + self.assertTrue('old' in self.config.pick_apache_config()) + + self.config.version = (2, 4, 11) + self.config._openssl_version = '1.0.2m' + self.assertTrue('current' in self.config.pick_apache_config()) + + self.config._openssl_version = '1.0.2a' + self.assertTrue('old' in self.config.pick_apache_config()) + + def test_openssl_version_warns(self): + self.config._openssl_version = '1.0.2a' + self.assertEqual(self.config.openssl_version(), '1.0.2a') + + self.config._openssl_version = None + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Could not find ssl_module" in mock_log.call_args[0][0]) + + self.config._openssl_version = None + self.config.parser.modules['ssl_module'] = None + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Could not find ssl_module" in mock_log.call_args[0][0]) + + self.config.parser.modules['ssl_module'] = "/fake/path" + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + # Check that correct logger.warning was printed + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Unable to read" in mock_log.call_args[0][0]) + + contents_missing_openssl = b"these contents won't match the regex" + with mock.patch("certbot_apache._internal.configurator." + "ApacheConfigurator._open_module_file") as mock_omf: + mock_omf.return_value = contents_missing_openssl + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + # Check that correct logger.warning was printed + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Could not find OpenSSL" in mock_log.call_args[0][0]) + + def test_open_module_file(self): + mock_open = mock.mock_open(read_data="testing 12 3") + with mock.patch("six.moves.builtins.open", mock_open): + self.assertEqual(self.config._open_module_file("/nonsense/"), "testing 12 3") if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/tests/debian_test.py b/certbot-apache/tests/debian_test.py index 400e503fb..7a65bb7b1 100644 --- a/certbot-apache/tests/debian_test.py +++ b/certbot-apache/tests/debian_test.py @@ -61,8 +61,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): def test_deploy_cert_enable_new_vhost(self): # Create ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None self.assertFalse(ssl_vhost.enabled) self.config.deploy_cert( "encryption-example.demo", "example/cert.pem", "example/key.pem", @@ -92,8 +92,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) self.config = self.mock_deploy_cert(self.config) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] @@ -128,8 +128,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) self.config = self.mock_deploy_cert(self.config) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] @@ -143,8 +143,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 7)) self.config = self.mock_deploy_cert(self.config) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] @@ -157,7 +157,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling_enable_mod(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True # This will create an ssl vhost for certbot.demo @@ -169,7 +169,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ensure_http_header_enable_mod(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None mock_exe.return_value = True # This will create an ssl vhost for certbot.demo diff --git a/certbot-apache/tests/fedora_test.py b/certbot-apache/tests/fedora_test.py index cb1614278..7f1d6526f 100644 --- a/certbot-apache/tests/fedora_test.py +++ b/certbot-apache/tests/fedora_test.py @@ -120,7 +120,7 @@ class MultipleVhostsTestFedora(util.ApacheTest): return mod_val return "" mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() + self.config.parser.modules = {} self.config.parser.variables = {} with mock.patch("certbot.util.get_os_info") as mock_osi: diff --git a/certbot-apache/tests/gentoo_test.py b/certbot-apache/tests/gentoo_test.py index fb5d192d0..3b35dde54 100644 --- a/certbot-apache/tests/gentoo_test.py +++ b/certbot-apache/tests/gentoo_test.py @@ -117,7 +117,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest): return mod_val return None # pragma: no cover mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() + self.config.parser.modules = {} with mock.patch("certbot.util.get_os_info") as mock_osi: # Make sure we have the have the Gentoo httpd constants diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 85b17ca28..7d9019702 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -40,8 +40,8 @@ class ApacheHttp01Test(util.ApacheTest): modules = ["ssl", "rewrite", "authz_core", "authz_host"] for mod in modules: - self.config.parser.modules.add("mod_{0}.c".format(mod)) - self.config.parser.modules.add(mod + "_module") + self.config.parser.modules["mod_{0}.c".format(mod)] = None + self.config.parser.modules[mod + "_module"] = None from certbot_apache._internal.http_01 import ApacheHttp01 self.http = ApacheHttp01(self.config) @@ -52,24 +52,24 @@ class ApacheHttp01Test(util.ApacheTest): @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod") def test_enable_modules_apache_2_2(self, mock_enmod): self.config.version = (2, 2) - self.config.parser.modules.remove("authz_host_module") - self.config.parser.modules.remove("mod_authz_host.c") + del self.config.parser.modules["authz_host_module"] + del self.config.parser.modules["mod_authz_host.c"] enmod_calls = self.common_enable_modules_test(mock_enmod) self.assertEqual(enmod_calls[0][0][0], "authz_host") @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod") def test_enable_modules_apache_2_4(self, mock_enmod): - self.config.parser.modules.remove("authz_core_module") - self.config.parser.modules.remove("mod_authz_core.c") + del self.config.parser.modules["authz_core_module"] + del self.config.parser.modules["mod_authz_host.c"] enmod_calls = self.common_enable_modules_test(mock_enmod) self.assertEqual(enmod_calls[0][0][0], "authz_core") def common_enable_modules_test(self, mock_enmod): """Tests enabling mod_rewrite and other modules.""" - self.config.parser.modules.remove("rewrite_module") - self.config.parser.modules.remove("mod_rewrite.c") + del self.config.parser.modules["rewrite_module"] + del self.config.parser.modules["mod_rewrite.c"] self.http.prepare_http01_modules() diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py index f5a0a3d11..299eb4567 100644 --- a/certbot-apache/tests/parser_test.py +++ b/certbot-apache/tests/parser_test.py @@ -114,7 +114,7 @@ class BasicParserTest(util.ParserTest): """ from certbot_apache._internal.parser import get_aug_path # This makes sure that find_dir will work - self.parser.modules.add("mod_ssl.c") + self.parser.modules["mod_ssl.c"] = "/fake/path" self.parser.add_dir_to_ifmodssl( get_aug_path(self.parser.loc["default"]), @@ -128,7 +128,7 @@ class BasicParserTest(util.ParserTest): def test_add_dir_to_ifmodssl_multiple(self): from certbot_apache._internal.parser import get_aug_path # This makes sure that find_dir will work - self.parser.modules.add("mod_ssl.c") + self.parser.modules["mod_ssl.c"] = "/fake/path" self.parser.add_dir_to_ifmodssl( get_aug_path(self.parser.loc["default"]), @@ -260,7 +260,7 @@ class BasicParserTest(util.ParserTest): expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443", "example_path": "Documents/path"} - self.parser.modules = set() + self.parser.modules = {} with mock.patch( "certbot_apache._internal.parser.ApacheParser.parse_file") as mock_parse: self.parser.update_runtime_variables() @@ -282,7 +282,7 @@ class BasicParserTest(util.ParserTest): os.path.dirname(self.parser.loc["root"])) mock_cfg.return_value = inc_val - self.parser.modules = set() + self.parser.modules = {} with mock.patch( "certbot_apache._internal.parser.ApacheParser.parse_file") as mock_parse: diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index ccd0b274d..4994103f3 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -85,7 +85,8 @@ def get_apache_configurator( config_dir, work_dir, version=(2, 4, 7), os_info="generic", conf_vhost_path=None, - use_parsernode=False): + use_parsernode=False, + openssl_version="1.1.1a"): """Create an Apache Configurator with the specified options. :param conf: Function that returns binary paths. self.conf in Configurator @@ -118,7 +119,8 @@ def get_apache_configurator( except KeyError: config_class = configurator.ApacheConfigurator config = config_class(config=mock_le_config, name="apache", - version=version, use_parsernode=use_parsernode) + version=version, use_parsernode=use_parsernode, + openssl_version=openssl_version) if not conf_vhost_path: config_class.OS_DEFAULTS["vhost_root"] = vhost_path else: diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index c35af0d0f..2f551c8ab 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -6,6 +6,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added +* Turn off session tickets for apache plugin by default when appropriate. * Added serial number of certificate to the output of `certbot certificates` * Expose two new environment variables in the authenticator and cleanup scripts used by the `manual` plugin: `CERTBOT_REMAINING_CHALLENGES` is equal to the number of challenges @@ -31,9 +32,9 @@ More details about these changes can be found on our GitHub repo. ### Added -* Added certbot.ocsp Certbot's API. The certbot.ocsp module can be used to +* Added certbot.ocsp Certbot's API. The certbot.ocsp module can be used to determine the OCSP status of certificates. -* Don't verify the existing certificate in HTTP01Response.simple_verify, for +* Don't verify the existing certificate in HTTP01Response.simple_verify, for compatibility with the real-world ACME challenge checks. * Added support for `$hostname` in nginx `server_name` directive @@ -42,7 +43,7 @@ More details about these changes can be found on our GitHub repo. * Certbot will now renew certificates early if they have been revoked according to OCSP. * Fix acme module warnings when response Content-Type includes params (e.g. charset). -* Fixed issue where webroot plugin would incorrectly raise `Read-only file system` +* Fixed issue where webroot plugin would incorrectly raise `Read-only file system` error when creating challenge directories (issue #7165). ### Fixed diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 9af39e8bb..9c3b95a31 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -58,6 +58,22 @@ if [ $? -ne 0 ] ; then FAIL=1 fi +# Check that ssl_module detection is working on various systems +if [ "$OS_TYPE" = "ubuntu" ] ; then + MOD_SSL_LOCATION="/usr/lib/apache2/modules/mod_ssl.so" + APACHE_NAME=apache2ctl +elif [ "$OS_TYPE" = "centos" ]; then + MOD_SSL_LOCATION="/etc/httpd/modules/mod_ssl.so" + APACHE_NAME=httpd +fi +OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9) +APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8) +"$PYTHON_NAME" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" +if [ $? -ne 0 ] ; then + FAIL=1 +fi + + if [ "$OS_TYPE" = "ubuntu" ] ; then export SERVER="$BOULDER_URL" "$VENV_PATH/bin/tox" -e apacheconftest diff --git a/tests/letstest/scripts/test_openssl_version.py b/tests/letstest/scripts/test_openssl_version.py new file mode 100644 index 000000000..c55441c5d --- /dev/null +++ b/tests/letstest/scripts/test_openssl_version.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# Test script for OpenSSL version checking +from distutils.version import LooseVersion +import sys + + +def main(openssl_version, apache_version): + if not openssl_version.strip(): + raise Exception("No OpenSSL version found.") + if not apache_version.strip(): + raise Exception("No Apache version found.") + conf_file_location = "/etc/letsencrypt/options-ssl-apache.conf" + with open(conf_file_location) as f: + contents = f.read() + if LooseVersion(apache_version.strip()) < LooseVersion('2.4.11') or \ + LooseVersion(openssl_version.strip()) < LooseVersion('1.0.2l'): + # should be old version + # assert SSLSessionTickets not in conf file + if "SSLSessionTickets" in contents: + raise Exception("Apache or OpenSSL version is too old, " + "but SSLSessionTickets is set.") + else: + # should be current version + # assert SSLSessionTickets in conf file + if "SSLSessionTickets" not in contents: + raise Exception("Apache and OpenSSL versions are sufficiently new, " + "but SSLSessionTickets is not set.") + +if __name__ == '__main__': + main(*sys.argv[1:]) From 1285297b23e445c10dfac481b02447d4cc2cf6f6 Mon Sep 17 00:00:00 2001 From: m0namon <51840727+m0namon@users.noreply.github.com> Date: Mon, 23 Mar 2020 17:05:22 -0700 Subject: [PATCH 50/72] [Apache v2] Load apacheconfig tree and gate related tests (#7710) * Load apacheconfig dependency, gate behind flag * Bump apacheconfig dependency to latest version and install dev version of apache for coverage tests * Move augeasnode_test tests to more generic parsernode_test * Revert "Move augeasnode_test tests to more generic parsernode_test" This reverts commit 6bb986ef786b9d68bb72776bde66e6572cf505a9. * Mock AugeasNode into DualNode's place, and run augeasnode tests exclusively on AugeasNode * Don't calculate coverage for skeleton functions * clean up helper function in augeasnode_test --- .../certbot_apache/_internal/apacheparser.py | 11 ++-- .../certbot_apache/_internal/configurator.py | 23 +++++-- certbot-apache/setup.py | 2 +- certbot-apache/tests/augeasnode_test.py | 65 ++++++++++++------- .../tests/parsernode_configurator_test.py | 7 ++ tools/dev_constraints.txt | 2 +- tools/oldest_constraints.txt | 2 +- tox.ini | 2 + 8 files changed, 76 insertions(+), 38 deletions(-) diff --git a/certbot-apache/certbot_apache/_internal/apacheparser.py b/certbot-apache/certbot_apache/_internal/apacheparser.py index 77f4517fe..c7b723ae6 100644 --- a/certbot-apache/certbot_apache/_internal/apacheparser.py +++ b/certbot-apache/certbot_apache/_internal/apacheparser.py @@ -73,7 +73,7 @@ class ApacheDirectiveNode(ApacheParserNode): self.metadata == other.metadata) return False - def set_parameters(self, _parameters): + def set_parameters(self, _parameters): # pragma: no cover """Sets the parameters for DirectiveNode""" return @@ -97,7 +97,8 @@ class ApacheBlockNode(ApacheDirectiveNode): self.metadata == other.metadata) return False - def add_child_block(self, name, parameters=None, position=None): # pylint: disable=unused-argument + # pylint: disable=unused-argument + def add_child_block(self, name, parameters=None, position=None): # pragma: no cover """Adds a new BlockNode to the sequence of children""" new_block = ApacheBlockNode(name=assertions.PASS, parameters=assertions.PASS, @@ -107,7 +108,8 @@ class ApacheBlockNode(ApacheDirectiveNode): self.children += (new_block,) return new_block - def add_child_directive(self, name, parameters=None, position=None): # pylint: disable=unused-argument + # pylint: disable=unused-argument + def add_child_directive(self, name, parameters=None, position=None): # pragma: no cover """Adds a new DirectiveNode to the sequence of children""" new_dir = ApacheDirectiveNode(name=assertions.PASS, parameters=assertions.PASS, @@ -144,7 +146,8 @@ class ApacheBlockNode(ApacheDirectiveNode): filepath=assertions.PASS, metadata=self.metadata)] - def find_comments(self, comment, exact=False): # pylint: disable=unused-argument + # pylint: disable=unused-argument + def find_comments(self, comment, exact=False): # pragma: no cover """Recursive search of DirectiveNodes from the sequence of children""" return [ApacheCommentNode(comment=assertions.PASS, ancestor=self, diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index fc386404d..86d579954 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -12,6 +12,11 @@ import time import six import zope.component import zope.interface +try: + import apacheconfig + HAS_APACHECONFIG = True +except ImportError: # pragma: no cover + HAS_APACHECONFIG = False from acme import challenges from acme.magic_typing import DefaultDict @@ -430,11 +435,17 @@ class ApacheConfigurator(common.Installer): def get_parsernode_root(self, metadata): """Initializes the ParserNode parser root instance.""" - apache_vars = dict() - apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) - apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) - apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) - metadata["apache_vars"] = apache_vars + if HAS_APACHECONFIG: + apache_vars = dict() + apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) + apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) + apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) + metadata["apache_vars"] = apache_vars + + with open(self.parser.loc["root"]) as f: + with apacheconfig.make_loader(writable=True, + **apacheconfig.flavors.NATIVE_APACHE) as loader: + metadata["ac_ast"] = loader.loads(f.read()) return dualparser.DualBlockNode( name=assertions.PASS, @@ -974,7 +985,7 @@ class ApacheConfigurator(common.Installer): """ v1_vhosts = self.get_virtual_hosts_v1() - if self.USE_PARSERNODE: + if self.USE_PARSERNODE and HAS_APACHECONFIG: v2_vhosts = self.get_virtual_hosts_v2() for v1_vh in v1_vhosts: diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 4ec1d0a9c..14300370a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -19,7 +19,7 @@ install_requires = [ ] dev_extras = [ - 'apacheconfig>=0.3.1', + 'apacheconfig>=0.3.2', ] class PyTest(TestCommand): diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 8417bc283..4468a838e 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -1,13 +1,25 @@ """Tests for AugeasParserNode classes""" import mock +import os +import unittest import util from certbot import errors from certbot_apache._internal import assertions +from certbot_apache._internal import augeasparser +def _get_augeasnode_mock(filepath): + """ Helper function for mocking out DualNode instance with an AugeasNode """ + def augeasnode_mock(metadata): + return augeasparser.AugeasBlockNode( + name=assertions.PASS, + ancestor=None, + filepath=filepath, + metadata=metadata) + return augeasnode_mock class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public-methods """Test AugeasParserNode using available test configurations""" @@ -15,8 +27,11 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- def setUp(self): # pylint: disable=arguments-differ super(AugeasParserNodeTest, self).setUp() - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, use_parsernode=True) + with mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.get_parsernode_root") as mock_parsernode: + mock_parsernode.side_effect = _get_augeasnode_mock( + os.path.join(self.config_path, "apache2.conf")) + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, use_parsernode=True) self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") @@ -110,7 +125,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- name=servernames[0].name, parameters=["test", "setting", "these"], ancestor=assertions.PASS, - metadata=servernames[0].primary.metadata + metadata=servernames[0].metadata ) self.assertTrue(mock_set.called) self.assertEqual( @@ -145,26 +160,26 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.assertTrue(found) def test_add_child_comment(self): - newc = self.config.parser_root.primary.add_child_comment("The content") + newc = self.config.parser_root.add_child_comment("The content") comments = self.config.parser_root.find_comments("The content") self.assertEqual(len(comments), 1) self.assertEqual( newc.metadata["augeaspath"], - comments[0].primary.metadata["augeaspath"] + comments[0].metadata["augeaspath"] ) self.assertEqual(newc.comment, comments[0].comment) def test_delete_child(self): - listens = self.config.parser_root.primary.find_directives("Listen") + listens = self.config.parser_root.find_directives("Listen") self.assertEqual(len(listens), 1) - self.config.parser_root.primary.delete_child(listens[0]) + self.config.parser_root.delete_child(listens[0]) - listens = self.config.parser_root.primary.find_directives("Listen") + listens = self.config.parser_root.find_directives("Listen") self.assertEqual(len(listens), 0) def test_delete_child_not_found(self): listen = self.config.parser_root.find_directives("Listen")[0] - listen.primary.metadata["augeaspath"] = "/files/something/nonexistent" + listen.metadata["augeaspath"] = "/files/something/nonexistent" self.assertRaises( errors.PluginError, @@ -177,10 +192,10 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "NewBlock", ["first", "second"] ) - rpath, _, directive = nb.primary.metadata["augeaspath"].rpartition("/") + rpath, _, directive = nb.metadata["augeaspath"].rpartition("/") self.assertEqual( rpath, - self.config.parser_root.primary.metadata["augeaspath"] + self.config.parser_root.metadata["augeaspath"] ) self.assertTrue(directive.startswith("NewBlock")) @@ -189,8 +204,8 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "Beginning", position=0 ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Get first child first = parser.aug.match("{}/*[1]".format(root_path)) self.assertTrue(first[0].endswith("Beginning")) @@ -199,8 +214,8 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.config.parser_root.add_child_block( "VeryLast", ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Get last child last = parser.aug.match("{}/*[last()]".format(root_path)) self.assertTrue(last[0].endswith("VeryLast")) @@ -210,8 +225,8 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "VeryLastAlt", position=99999 ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Get last child last = parser.aug.match("{}/*[last()]".format(root_path)) self.assertTrue(last[0].endswith("VeryLastAlt")) @@ -221,15 +236,15 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "Middle", position=5 ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Augeas indices start at 1 :( middle = parser.aug.match("{}/*[6]".format(root_path)) self.assertTrue(middle[0].endswith("Middle")) def test_add_child_block_existing_name(self): - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # There already exists a single VirtualHost in the base config new_block = parser.aug.match("{}/VirtualHost[2]".format(root_path)) self.assertEqual(len(new_block), 0) @@ -238,7 +253,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- ) new_block = parser.aug.match("{}/VirtualHost[2]".format(root_path)) self.assertEqual(len(new_block), 1) - self.assertTrue(vh.primary.metadata["augeaspath"].endswith("VirtualHost[2]")) + self.assertTrue(vh.metadata["augeaspath"].endswith("VirtualHost[2]")) def test_node_init_error_bad_augeaspath(self): from certbot_apache._internal.augeasparser import AugeasBlockNode @@ -283,7 +298,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.assertEqual(len(dirs), 1) self.assertEqual(dirs[0].parameters, ("with", "parameters")) # The new directive was added to the very first line of the config - self.assertTrue(dirs[0].primary.metadata["augeaspath"].endswith("[1]")) + self.assertTrue(dirs[0].metadata["augeaspath"].endswith("[1]")) def test_add_child_directive_exception(self): self.assertRaises( @@ -313,6 +328,6 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.assertTrue(nonmacro_test) def test_find_ancestors_bad_path(self): - self.config.parser_root.primary.metadata["augeaspath"] = "" - ancs = self.config.parser_root.primary.find_ancestors("Anything") + self.config.parser_root.metadata["augeaspath"] = "" + ancs = self.config.parser_root.find_ancestors("Anything") self.assertEqual(len(ancs), 0) diff --git a/certbot-apache/tests/parsernode_configurator_test.py b/certbot-apache/tests/parsernode_configurator_test.py index 67d65995a..afaaa6e43 100644 --- a/certbot-apache/tests/parsernode_configurator_test.py +++ b/certbot-apache/tests/parsernode_configurator_test.py @@ -5,7 +5,14 @@ import mock import util +try: + import apacheconfig + HAS_APACHECONFIG = True +except ImportError: # pragma: no cover + HAS_APACHECONFIG = False + +@unittest.skipIf(not HAS_APACHECONFIG, reason='Tests require apacheconfig dependency') class ConfiguratorParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public-methods """Test AugeasParserNode using available test configurations""" diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index cfa036435..6e692841b 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -3,7 +3,7 @@ # Some dev package versions specified here may be overridden by higher level constraints # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). alabaster==0.7.10 -apacheconfig==0.3.1 +apacheconfig==0.3.2 apipkg==1.4 appnope==0.1.0 asn1crypto==0.22.0 diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 85d058796..402f3fef1 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -39,7 +39,7 @@ pytz==2012rc0 google-api-python-client==1.5.5 # Our setup.py constraints -apacheconfig==0.3.1 +apacheconfig==0.3.2 cloudflare==1.5.1 cryptography==1.2.3 parsedatetime==1.3 diff --git a/tox.ini b/tox.ini index 3903cdf45..8aa4bfbf2 100644 --- a/tox.ini +++ b/tox.ini @@ -120,12 +120,14 @@ setenv = basepython = python2.7 commands = {[base]install_packages} + {[base]pip_install} certbot-apache[dev] python tox.cover.py [testenv:py37-cover] basepython = python3.7 commands = {[base]install_packages} + {[base]pip_install} certbot-apache[dev] python tox.cover.py [testenv:lint] From e4a0edc7af05443448082a771e5cb981fa6544ed Mon Sep 17 00:00:00 2001 From: alexzorin Date: Wed, 25 Mar 2020 01:02:53 +1100 Subject: [PATCH 51/72] Add a 10-second timeout to OCSP queries. (#7860) * Add a 10-second timeout to OCSP queries. Closes #7859 * Update CHANGELOG * Fix test --- certbot/CHANGELOG.md | 1 + certbot/certbot/ocsp.py | 23 +++++++++++++---------- certbot/tests/ocsp_test.py | 4 ++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 2f551c8ab..c6f6c95a3 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -25,6 +25,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). `resource` field in any requests or the `type` field when responding to challenges. * Fix nginx plugin crash when non-ASCII configuration file is being read (instead, the user will be warned that UTF-8 must be used). +* Fix hanging OCSP queries during revocation checking - added a 10 second timeout. More details about these changes can be found on our GitHub repo. diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 9799c675c..1d5611b64 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -70,12 +70,13 @@ class RevocationChecker(object): """ return self.ocsp_revoked_by_paths(cert.cert_path, cert.chain_path) - def ocsp_revoked_by_paths(self, cert_path, chain_path): - # type: (str, str) -> bool + def ocsp_revoked_by_paths(self, cert_path, chain_path, timeout=10): + # type: (str, str, int) -> bool """Performs the OCSP revocation check :param str cert_path: Certificate filepath - :param str chain_path: Certificate chain filepath + :param str chain_path: Certificate chain + :param int timeout: Timeout (in seconds) for the OCSP query :returns: True if revoked; False if valid or the check failed or cert is expired. :rtype: bool @@ -96,11 +97,11 @@ class RevocationChecker(object): return False if self.use_openssl_binary: - return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url) - return _check_ocsp_cryptography(cert_path, chain_path, url) + return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url, timeout) + return _check_ocsp_cryptography(cert_path, chain_path, url, timeout) - def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url): - # type: (str, str, str, str) -> bool + def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url, timeout): + # type: (str, str, str, str, int) -> bool # jdkasten thanks "Bulletproof SSL and TLS - Ivan Ristic" for documenting this! cmd = ["openssl", "ocsp", "-no_nonce", @@ -110,6 +111,7 @@ class RevocationChecker(object): "-CAfile", chain_path, "-verify_other", chain_path, "-trust_other", + "-timeout", str(timeout), "-header"] + self.host_args(host) logger.debug("Querying OCSP for %s", cert_path) logger.debug(" ".join(cmd)) @@ -152,8 +154,8 @@ def _determine_ocsp_server(cert_path): return None, None -def _check_ocsp_cryptography(cert_path, chain_path, url): - # type: (str, str, str) -> bool +def _check_ocsp_cryptography(cert_path, chain_path, url, timeout): + # type: (str, str, str, int) -> bool # Retrieve OCSP response with open(chain_path, 'rb') as file_handler: issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) @@ -165,7 +167,8 @@ def _check_ocsp_cryptography(cert_path, chain_path, url): request_binary = request.public_bytes(serialization.Encoding.DER) try: response = requests.post(url, data=request_binary, - headers={'Content-Type': 'application/ocsp-request'}) + headers={'Content-Type': 'application/ocsp-request'}, + timeout=timeout) except requests.exceptions.RequestException: logger.info("OCSP check failed for %s (are we offline?)", cert_path, exc_info=True) return False diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 6b05a8d3c..9eb49e115 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -162,11 +162,11 @@ class OSCPTestCryptography(unittest.TestCase): @mock.patch('certbot.ocsp._determine_ocsp_server') @mock.patch('certbot.ocsp._check_ocsp_cryptography') - def test_ensure_cryptography_toggled(self, mock_revoke, mock_determine): + def test_ensure_cryptography_toggled(self, mock_check, mock_determine): mock_determine.return_value = ('http://example.com', 'example.com') self.checker.ocsp_revoked(self.cert_obj) - mock_revoke.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com') + mock_check.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com', 10) def test_revoke(self): with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL): From 6df90d17aeb3865ceda8ae30372cb1416a1a6c3b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 25 Mar 2020 14:50:13 -0700 Subject: [PATCH 52/72] Wait 5 minutes for boulder to start. (#7864) --- certbot-ci/certbot_integration_tests/utils/acme_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index b14bd4a32..4b20d5acc 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -170,7 +170,7 @@ class ACMEServer(object): # Wait for the ACME CA server to be up. print('=> Waiting for boulder instance to respond...') - misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=240) + misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300) # Configure challtestsrv to answer any A record request with ip of the docker host. response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), From dbda499b08722da8311fb2352bf423318772192e Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 31 Mar 2020 12:12:14 -0700 Subject: [PATCH 53/72] Remove interactive redirect ask (#7861) Fixes #7594. Removes the code asking interactively if the user would like to add a redirect. * Remove interactive redirect ask * display.enhancements is no longer used, so remove it. * update changelog * remove references to removed display.enhancements * add redirect_default flag to enhance_config to conditionally set default for redirect value * Update default in help text. --- certbot/CHANGELOG.md | 2 +- certbot/certbot/_internal/cli/__init__.py | 6 +- certbot/certbot/_internal/client.py | 14 ++--- .../certbot/_internal/display/enhancements.py | 63 ------------------- certbot/certbot/_internal/main.py | 2 +- certbot/tests/client_test.py | 19 ++---- certbot/tests/display/enhancements_test.py | 57 ----------------- 7 files changed, 14 insertions(+), 149 deletions(-) delete mode 100644 certbot/certbot/_internal/display/enhancements.py delete mode 100644 certbot/tests/display/enhancements_test.py diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index c6f6c95a3..88a4ab190 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -17,7 +17,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* +* Stop asking interactively if the user would like to add a redirect. ### Fixed diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index 96dfb163e..2f83edcc8 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -321,12 +321,14 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): "--redirect", action="store_true", dest="redirect", default=flag_default("redirect"), help="Automatically redirect all HTTP traffic to HTTPS for the newly " - "authenticated vhost. (default: Ask)") + "authenticated vhost. (default: redirect enabled for install and run, " + "disabled for enhance)") helpful.add( "security", "--no-redirect", action="store_false", dest="redirect", default=flag_default("redirect"), help="Do not automatically redirect all HTTP traffic to HTTPS for the newly " - "authenticated vhost. (default: Ask)") + "authenticated vhost. (default: redirect enabled for install and run, " + "disabled for enhance)") helpful.add( ["security", "enhance"], "--hsts", action="store_true", dest="hsts", default=flag_default("hsts"), diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index 526f4200f..a9bf946cc 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -28,7 +28,6 @@ from certbot._internal import constants from certbot._internal import eff from certbot._internal import error_handler from certbot._internal import storage -from certbot._internal.display import enhancements from certbot._internal.plugins import selection as plugin_selection from certbot.compat import os from certbot.display import ops as display_ops @@ -521,12 +520,13 @@ class Client(object): # sites may have been enabled / final cleanup self.installer.restart() - def enhance_config(self, domains, chain_path, ask_redirect=True): + def enhance_config(self, domains, chain_path, redirect_default=True): """Enhance the configuration. :param list domains: list of domains to configure :param chain_path: chain file path :type chain_path: `str` or `None` + :param redirect_default: boolean value that the "redirect" flag should default to :raises .errors.Error: if no installer is specified in the client. @@ -548,14 +548,8 @@ class Client(object): for config_name, enhancement_name, option in enhancement_info: config_value = getattr(self.config, config_name) if enhancement_name in supported: - if ask_redirect: - if config_name == "redirect" and config_value is None: - config_value = enhancements.ask(enhancement_name) - if not config_value: - logger.warning("Future versions of Certbot will automatically " - "configure the webserver so that all requests redirect to secure " - "HTTPS access. You can control this behavior and disable this " - "warning with the --redirect and --no-redirect flags.") + if config_name == "redirect" and config_value is None: + config_value = redirect_default if config_value: self.apply_enhancement(domains, enhancement_name, option) enhanced = True diff --git a/certbot/certbot/_internal/display/enhancements.py b/certbot/certbot/_internal/display/enhancements.py deleted file mode 100644 index ce6470708..000000000 --- a/certbot/certbot/_internal/display/enhancements.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Certbot Enhancement Display""" -import logging - -import zope.component - -from certbot import errors -from certbot import interfaces -from certbot.display import util as display_util - -logger = logging.getLogger(__name__) - -# Define a helper function to avoid verbose code -util = zope.component.getUtility - - -def ask(enhancement): - """Display the enhancement to the user. - - :param str enhancement: One of the - :const:`~certbot.plugins.enhancements.ENHANCEMENTS` enhancements - - :returns: True if feature is desired, False otherwise - :rtype: bool - - :raises .errors.Error: if the enhancement provided is not supported - - """ - try: - # Call the appropriate function based on the enhancement - return DISPATCH[enhancement]() - except KeyError: - logger.error("Unsupported enhancement given to ask(): %s", enhancement) - raise errors.Error("Unsupported Enhancement") - - -def redirect_by_default(): - """Determines whether the user would like to redirect to HTTPS. - - :returns: True if redirect is desired, False otherwise - :rtype: bool - - """ - choices = [ - ("No redirect", "Make no further changes to the webserver configuration."), - ("Redirect", "Make all requests redirect to secure HTTPS access. " - "Choose this for new sites, or if you're confident your site works on HTTPS. " - "You can undo this change by editing your web server's configuration."), - ] - - code, selection = util(interfaces.IDisplay).menu( - "Please choose whether or not to redirect HTTP traffic to HTTPS, removing HTTP access.", - choices, default=1, - cli_flag="--redirect / --no-redirect", force_interactive=True) - - if code != display_util.OK: - return False - - return selection == 1 - - -DISPATCH = { - "redirect": redirect_by_default -} diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 4a57dd78d..94ccb8b5e 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -927,7 +927,7 @@ def enhance(config, plugins): config.chain_path = lineage.chain_path if oldstyle_enh: le_client = _init_le_client(config, authenticator=None, installer=installer) - le_client.enhance_config(domains, config.chain_path, ask_redirect=False) + le_client.enhance_config(domains, config.chain_path, redirect_default=False) if enhancements.are_requested(config): enhancements.enable(lineage, domains, installer, config) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 7232ed84b..3e0e5b212 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -577,8 +577,7 @@ class EnhanceConfigTest(ClientTestCommon): self.assertRaises( errors.Error, self.client.enhance_config, [self.domain], None) - @mock.patch("certbot._internal.client.enhancements") - def test_unsupported(self, mock_enhancements): + def test_unsupported(self): self.client.installer = mock.MagicMock() self.client.installer.supported_enhancements.return_value = [] @@ -588,7 +587,6 @@ class EnhanceConfigTest(ClientTestCommon): self.client.enhance_config([self.domain], None) self.assertEqual(mock_logger.warning.call_count, 1) self.client.installer.enhance.assert_not_called() - mock_enhancements.ask.assert_not_called() @mock.patch("certbot._internal.client.logger") def test_already_exists_header(self, mock_log): @@ -612,14 +610,11 @@ class EnhanceConfigTest(ClientTestCommon): self._test_with_already_existing() self.assertFalse(mock_log.warning.called) - @mock.patch("certbot._internal.client.enhancements.ask") @mock.patch("certbot._internal.client.logger") - def test_warn_redirect(self, mock_log, mock_ask): + def test_no_warn_redirect(self, mock_log): self.config.redirect = None - mock_ask.return_value = False - self._test_with_already_existing() - self.assertTrue(mock_log.warning.called) - self.assertTrue("disable" in mock_log.warning.call_args[0][0]) + self._test_with_all_supported() + self.assertFalse(mock_log.warning.called) def test_no_ask_hsts(self): self.config.hsts = True @@ -670,12 +665,6 @@ class EnhanceConfigTest(ClientTestCommon): self.client.installer = installer self._test_error_with_rollback() - @mock.patch("certbot._internal.client.enhancements.ask") - def test_ask(self, mock_ask): - self.config.redirect = None - mock_ask.return_value = True - self._test_with_all_supported() - def _test_error_with_rollback(self): self._test_error() self.assertTrue(self.client.installer.restart.called) diff --git a/certbot/tests/display/enhancements_test.py b/certbot/tests/display/enhancements_test.py deleted file mode 100644 index edace29b1..000000000 --- a/certbot/tests/display/enhancements_test.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Module for enhancement UI.""" -import logging -import unittest - -import mock - -from certbot import errors -from certbot.display import util as display_util - - -class AskTest(unittest.TestCase): - """Test the ask method.""" - def setUp(self): - logging.disable(logging.CRITICAL) - - def tearDown(self): - logging.disable(logging.NOTSET) - - @classmethod - def _call(cls, enhancement): - from certbot._internal.display.enhancements import ask - return ask(enhancement) - - @mock.patch("certbot._internal.display.enhancements.util") - def test_redirect(self, mock_util): - mock_util().menu.return_value = (display_util.OK, 1) - self.assertTrue(self._call("redirect")) - - def test_key_error(self): - self.assertRaises(errors.Error, self._call, "unknown_enhancement") - - -class RedirectTest(unittest.TestCase): - """Test the redirect_by_default method.""" - @classmethod - def _call(cls): - from certbot._internal.display.enhancements import redirect_by_default - return redirect_by_default() - - @mock.patch("certbot._internal.display.enhancements.util") - def test_secure(self, mock_util): - mock_util().menu.return_value = (display_util.OK, 1) - self.assertTrue(self._call()) - - @mock.patch("certbot._internal.display.enhancements.util") - def test_cancel(self, mock_util): - mock_util().menu.return_value = (display_util.CANCEL, 1) - self.assertFalse(self._call()) - - @mock.patch("certbot._internal.display.enhancements.util") - def test_easy(self, mock_util): - mock_util().menu.return_value = (display_util.OK, 0) - self.assertFalse(self._call()) - - -if __name__ == "__main__": - unittest.main() # pragma: no cover From bc3088121b74158ca4b24bc380d043f187196957 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 31 Mar 2020 22:12:42 +0200 Subject: [PATCH 54/72] Add a step to check powershell version in vs2017-win2016 (#7870) Following discussion at https://github.com/certbot/certbot/pull/7539#issuecomment-572318805, this PR adds a check for Powershell version: we expect that the `vs2017-win2016` node that will test the installer has Powershell 5.x, and nothing else. This ensure that at least one node of the pipeline is testing the installer with the lowest Powershell version supported by Certbot. One full pipeline success can be seen here: https://dev.azure.com/adferrand/certbot/_build/results?buildId=713 I also create on purpose a failing pipeline, that would check that Powershell 6.x is installed. Its result can be seen here: https://dev.azure.com/adferrand/certbot/_build/results?buildId=714 --- .azure-pipelines/templates/installer-tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.azure-pipelines/templates/installer-tests.yml b/.azure-pipelines/templates/installer-tests.yml index ea2101792..ebadcb2dc 100644 --- a/.azure-pipelines/templates/installer-tests.yml +++ b/.azure-pipelines/templates/installer-tests.yml @@ -31,6 +31,13 @@ jobs: pool: vmImage: $(imageName) steps: + - powershell: | + $currentVersion = $PSVersionTable.PSVersion + if ($currentVersion.Major -ne 5) { + throw "Powershell version is not 5.x" + } + condition: eq(variables['imageName'], 'vs2017-win2016') + displayName: Check Powershell 5.x is used in vs2017-win2016 - task: UsePythonVersion@0 inputs: versionSpec: 3.8 From 4ca86d9482cbb9bca04335bfb7dbd892cefe91c8 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Thu, 2 Apr 2020 08:53:58 +1100 Subject: [PATCH 55/72] acme: socket timeout for HTTP standalone servers (#7388) * acme: socket timeout for HTTP standalone servers Adds a default 30 second timeout to the StreamRequestHandler for clients connecting to standalone HTTP-01 servers. This should prevent most cases of an idle client connection from preventing the standalone server from shutting down. Fixes #7386 * use idiomatic kwargs default value * move HTTP01Server lower to fix mypy forward ref. * fix test crash on macOS due to socket double-close * maybe its not an OSError? * disable coverage check on useless branch --- acme/acme/standalone.py | 10 ++++++---- acme/tests/standalone_test.py | 22 ++++++++++++++++++++++ certbot/CHANGELOG.md | 2 ++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 52ac07915..7a61ba868 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -176,10 +176,10 @@ class HTTPServer(BaseHTTPServer.HTTPServer): class HTTP01Server(HTTPServer, ACMEServerMixin): """HTTP01 Server.""" - def __init__(self, server_address, resources, ipv6=False): + def __init__(self, server_address, resources, ipv6=False, timeout=30): HTTPServer.__init__( self, server_address, HTTP01RequestHandler.partial_init( - simple_http_resources=resources), ipv6=ipv6) + simple_http_resources=resources, timeout=timeout), ipv6=ipv6) class HTTP01DualNetworkedServers(BaseDualNetworkedServers): @@ -204,6 +204,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.simple_http_resources = kwargs.pop("simple_http_resources", set()) + self.timeout = kwargs.pop('timeout', 30) BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def log_message(self, format, *args): # pylint: disable=redefined-builtin @@ -253,7 +254,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.path) @classmethod - def partial_init(cls, simple_http_resources): + def partial_init(cls, simple_http_resources, timeout): """Partially initialize this handler. This is useful because `socketserver.BaseServer` takes @@ -262,7 +263,8 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """ return functools.partial( - cls, simple_http_resources=simple_http_resources) + cls, simple_http_resources=simple_http_resources, + timeout=timeout) class _BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler): diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 8c08ab89b..8face7c7b 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -85,6 +85,28 @@ class HTTP01ServerTest(unittest.TestCase): def test_http01_not_found(self): self.assertFalse(self._test_http01(add=False)) + def test_timely_shutdown(self): + from acme.standalone import HTTP01Server + server = HTTP01Server(('', 0), resources=set(), timeout=0.05) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.start() + + client = socket.socket() + client.connect(('localhost', server.socket.getsockname()[1])) + + stop_thread = threading.Thread(target=server.shutdown) + stop_thread.start() + server_thread.join(5.) + + is_hung = server_thread.is_alive() + try: + client.shutdown(socket.SHUT_RDWR) + except: # pragma: no cover, pylint: disable=bare-except + # may raise error because socket could already be closed + pass + + self.assertFalse(is_hung, msg='Server shutdown should not be hung') + @unittest.skipIf(not challenges.TLSALPN01.is_supported(), "pyOpenSSL too old") class TLSALPN01ServerTest(unittest.TestCase): diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 88a4ab190..99c1de63b 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -26,6 +26,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Fix nginx plugin crash when non-ASCII configuration file is being read (instead, the user will be warned that UTF-8 must be used). * Fix hanging OCSP queries during revocation checking - added a 10 second timeout. +* Standalone servers now have a default socket timeout of 30 seconds, fixing + cases where an idle connection can cause the standalone plugin to hang. More details about these changes can be found on our GitHub repo. From 5992d521e225fd3abf176f07bdc7753a9568526b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 7 Apr 2020 15:26:19 -0700 Subject: [PATCH 56/72] Print boulder logs when boulder setup fails (#7885) This is part of https://github.com/certbot/certbot/issues/7303. * Print boulder logs if boulder fails to start * Print description and fix command. * Change output to stderr. --- .../utils/acme_server.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 4b20d5acc..fb202005e 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -38,7 +38,7 @@ class ACMEServer(object): :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) :param list nodes: list of node names that will be setup by pytest xdist :param bool http_proxy: if False do not start the HTTP proxy - :param bool stdout: if True stream subprocesses stdout to standard stdout + :param bool stdout: if True stream all subprocesses stdout to standard stdout """ self._construct_acme_xdist(acme_server, nodes) @@ -165,17 +165,24 @@ class ACMEServer(object): os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'), join(instance_path, 'test/rate-limit-policies.yml')) - # Launch the Boulder server - self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path) + try: + # Launch the Boulder server + self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path) - # Wait for the ACME CA server to be up. - print('=> Waiting for boulder instance to respond...') - misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300) + # Wait for the ACME CA server to be up. + print('=> Waiting for boulder instance to respond...') + misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300) - # Configure challtestsrv to answer any A record request with ip of the docker host. - response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), - json={'ip': '10.77.77.1'}) - response.raise_for_status() + # Configure challtestsrv to answer any A record request with ip of the docker host. + response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), + json={'ip': '10.77.77.1'}) + response.raise_for_status() + except BaseException: + # If we failed to set up boulder, print its logs. + print('=> Boulder setup failed. Boulder logs are:') + process = self._launch_process(['docker-compose', 'logs'], cwd=instance_path, force_stderr=True) + process.wait() + raise print('=> Finished boulder instance deployment.') @@ -188,11 +195,12 @@ class ACMEServer(object): self._launch_process(command) print('=> Finished configuring the HTTP proxy.') - def _launch_process(self, command, cwd=os.getcwd(), env=None): + def _launch_process(self, command, cwd=os.getcwd(), env=None, force_stderr=False): """Launch silently a subprocess OS command""" if not env: env = os.environ - process = subprocess.Popen(command, stdout=self._stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env) + stdout = sys.stderr if force_stderr else self._stdout + process = subprocess.Popen(command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env) self._processes.append(process) return process From e9895d2ec69395cfcf1cfff35e7668b57e1da00f Mon Sep 17 00:00:00 2001 From: alexzorin Date: Wed, 8 Apr 2020 09:19:13 +1000 Subject: [PATCH 57/72] Fix fullchain parsing for CRLF chains (#7877) Fixes #7875 . After [this comment](https://github.com/certbot/certbot/issues/7875#issuecomment-608145208) and evaluating the options, I opted to go with `stricttextualmsg`, as required by RFC 8555. Reasoning is that the ACME v1 code path (via OpenSSL) produces a `fullchain_pem` which satisfies `stricttextualmsg`, so we don't need to be more generous than that. One downside of the `re` approach is that it doesn't seem capable of capturing repeating group matches. As a result, it matches each certificate individually, silently passing over any data in between the encapsulation boundaries, such as explanatory text, which is prohibited by RFC 8555. It would be ideal to raise an error when encountering such a non-conformant chain, but we'd need to create a mini-parser to do it, I think. * Fix fullchain parsing for CRLF chains. fullchain parsing now works in two passes: 1. A first pass which is generous with what it accepts - basically preeb(CERTIFICATE)+anything+posteb(CERTIFICATE). This determines the boundaries for each certificate. 2. A second pass which normalizes (by parsing and re-encoding) each certificate found in the first pass. * typo in docstring * remove redundant group in regex * can't use assertRaisesRegex until py27 is gone --- certbot/CHANGELOG.md | 2 ++ certbot/certbot/crypto_util.py | 33 +++++++++++++++++++++++++++---- certbot/tests/crypto_util_test.py | 18 ++++++++++++++++- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 99c1de63b..7813c4db3 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -28,6 +28,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Fix hanging OCSP queries during revocation checking - added a 10 second timeout. * Standalone servers now have a default socket timeout of 30 seconds, fixing cases where an idle connection can cause the standalone plugin to hang. +* Parsing of the RFC 8555 application/pem-certificate-chain now tolerates CRLF line + endings. This should fix interoperability with Buypass' services. More details about these changes can be found on our GitHub repo. diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index adb972f24..168531667 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -8,6 +8,7 @@ import hashlib import logging import warnings +import re # See https://github.com/pyca/cryptography/issues/4275 from cryptography import x509 # type: ignore from cryptography.exceptions import InvalidSignature @@ -478,6 +479,17 @@ def sha256sum(filename): sha256.update(file_d.read().encode('UTF-8')) return sha256.hexdigest() +# Finds one CERTIFICATE stricttextualmsg according to rfc7468#section-3. +# Does not validate the base64text - use crypto.load_certificate. +CERT_PEM_REGEX = re.compile( + b"""-----BEGIN CERTIFICATE-----\r? +.+?\r? +-----END CERTIFICATE-----\r? +""", + re.DOTALL # DOTALL (/s) because the base64text may include newlines +) + + def cert_and_chain_from_fullchain(fullchain_pem): """Split fullchain_pem into cert_pem and chain_pem @@ -486,11 +498,24 @@ def cert_and_chain_from_fullchain(fullchain_pem): :returns: tuple of string cert_pem and chain_pem :rtype: tuple + :raises errors.Error: If there are less than 2 certificates in the chain. + """ - cert = crypto.dump_certificate(crypto.FILETYPE_PEM, - crypto.load_certificate(crypto.FILETYPE_PEM, fullchain_pem)).decode() - chain = fullchain_pem[len(cert):].lstrip() - return (cert, chain) + # First pass: find the boundary of each certificate in the chain. + # TODO: This will silently skip over any "explanatory text" in between boundaries, + # which is prohibited by RFC8555. + certs = CERT_PEM_REGEX.findall(fullchain_pem.encode()) + if len(certs) < 2: + raise errors.Error("failed to parse fullchain into cert and chain: " + + "less than 2 certificates in chain") + + # Second pass: for each certificate found, parse it using OpenSSL and re-encode it, + # with the effect of normalizing any encoding variations (e.g. CRLF, whitespace). + certs_normalized = [crypto.dump_certificate(crypto.FILETYPE_PEM, + crypto.load_certificate(crypto.FILETYPE_PEM, cert)).decode() for cert in certs] + + # Since each normalized cert has a newline suffix, no extra newlines are required. + return (certs_normalized[0], "".join(certs_normalized[1:])) def get_serial_from_cert(cert_path): """Retrieve the serial number of a certificate from certificate path diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 1d642ae9e..d52e3acdb 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -376,17 +376,33 @@ class Sha256sumTest(unittest.TestCase): class CertAndChainFromFullchainTest(unittest.TestCase): """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" + def _parse_and_reencode_pem(self, cert_pem): + from OpenSSL import crypto + return crypto.dump_certificate(crypto.FILETYPE_PEM, + crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)).decode() + def test_cert_and_chain_from_fullchain(self): cert_pem = CERT.decode() chain_pem = cert_pem + SS_CERT.decode() fullchain_pem = cert_pem + chain_pem spacey_fullchain_pem = cert_pem + u'\n' + chain_pem + crlf_fullchain_pem = fullchain_pem.replace(u'\n', u'\r\n') + + # In the ACME v1 code path, the fullchain is constructed by loading cert+chain DERs + # and using OpenSSL to dump them, so here we confirm that OpenSSL is producing certs + # that will be parseable by cert_and_chain_from_fullchain. + acmev1_fullchain_pem = self._parse_and_reencode_pem(cert_pem) + \ + self._parse_and_reencode_pem(cert_pem) + self._parse_and_reencode_pem(SS_CERT.decode()) + from certbot.crypto_util import cert_and_chain_from_fullchain - for fullchain in (fullchain_pem, spacey_fullchain_pem): + for fullchain in (fullchain_pem, spacey_fullchain_pem, crlf_fullchain_pem, + acmev1_fullchain_pem): cert_out, chain_out = cert_and_chain_from_fullchain(fullchain) self.assertEqual(cert_out, cert_pem) self.assertEqual(chain_out, chain_pem) + self.assertRaises(errors.Error, cert_and_chain_from_fullchain, cert_pem) + if __name__ == '__main__': unittest.main() # pragma: no cover From 537bee09940abe064fe7d5daff25b7bb3748867c Mon Sep 17 00:00:00 2001 From: inejge Date: Thu, 9 Apr 2020 20:25:39 +0200 Subject: [PATCH 58/72] Add minimal proxy support for OCSP verification (#7892) Translate a proxy specified by an environment variable ("http_proxy" or "HTTP_PROXY") into options recognized by "openssl ocsp". Support is limited to HTTP proxies which don't require authentication. Fixes #6150 --- AUTHORS.md | 1 + certbot/CHANGELOG.md | 1 + certbot/certbot/ocsp.py | 20 ++++++++++++++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index f5b981b8e..4414076fc 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -103,6 +103,7 @@ Authors * [Henry Chen](https://github.com/henrychen95) * [Hugo van Kemenade](https://github.com/hugovk) * [Ingolf Becker](https://github.com/watercrossing) +* [Ivan Nejgebauer](https://github.com/inejge) * [Jaap Eldering](https://github.com/eldering) * [Jacob Hoffman-Andrews](https://github.com/jsha) * [Jacob Sachs](https://github.com/jsachs) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 7813c4db3..f61cdcfc7 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -14,6 +14,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). of all domains challenged for the current certificate. * Added TLS-ALPN-01 challenge support in the `acme` library. Support of this challenge in the Certbot client is planned to be added in a future release. +* Added minimal proxy support for OCSP verification. ### Changed diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 1d5611b64..863c5f163 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -21,6 +21,7 @@ from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import util +from certbot.compat.os import getenv from certbot.interfaces import RenewableCert # pylint: disable=unused-import try: @@ -102,17 +103,32 @@ class RevocationChecker(object): def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url, timeout): # type: (str, str, str, str, int) -> bool + # Minimal implementation of proxy selection logic as seen in, e.g., cURL + # Some things that won't work, but may well be in use somewhere: + # - username and password for proxy authentication + # - proxies accepting TLS connections + # - proxy exclusion through NO_PROXY + env_http_proxy = getenv('http_proxy') + env_HTTP_PROXY = getenv('HTTP_PROXY') + proxy_host = None + if env_http_proxy is not None or env_HTTP_PROXY is not None: + proxy_host = env_http_proxy if env_http_proxy is not None else env_HTTP_PROXY + if proxy_host is None: + url_opts = ["-url", url] + else: + if proxy_host.startswith('http://'): + proxy_host = proxy_host[len('http://'):] + url_opts = ["-host", proxy_host, "-path", url] # jdkasten thanks "Bulletproof SSL and TLS - Ivan Ristic" for documenting this! cmd = ["openssl", "ocsp", "-no_nonce", "-issuer", chain_path, "-cert", cert_path, - "-url", url, "-CAfile", chain_path, "-verify_other", chain_path, "-trust_other", "-timeout", str(timeout), - "-header"] + self.host_args(host) + "-header"] + self.host_args(host) + url_opts logger.debug("Querying OCSP for %s", cert_path) logger.debug(" ".join(cmd)) try: From 316e4640f86c7f48a48d194a77e9462d1f50bd34 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 9 Apr 2020 14:35:47 -0700 Subject: [PATCH 59/72] Upgrade the test farm tests to use Python 3 (#7876) Fixes #7857. * stop using urllib2 in test farm tests * use six for urllib instead * remove fabric lcd usage * correct lcd removal * remove fabric cd * convert some remote calls to v2 * move more cxns to v2 * get run working with prefix * get sudo commands working * remove final fabric v1 references including local * update requirements and README * add new venv to gitignore * update version used in travis * remove deploy_script unused kwargs * fix killboulder implementation so I can test creating a new boulder server * hardcode the gopath due to broken env manamagement in fabric2 * Update letstest readme * move the comment about hardcoding the ggopath * catch BaseException instead of Exception * work around fabric #2007 * use connections as context managers to ensure they're closed * remove reference to virtualenv --- .gitignore | 1 + .travis.yml | 8 +- tests/letstest/README.md | 10 +- tests/letstest/multitester.py | 235 ++++++++++++++++---------------- tests/letstest/requirements.txt | 4 +- 5 files changed, 133 insertions(+), 125 deletions(-) diff --git a/.gitignore b/.gitignore index 6dd422187..6505e716c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ tags tests/letstest/letest-*/ tests/letstest/*.pem tests/letstest/venv/ +tests/letstest/venv3/ .venv diff --git a/.travis.yml b/.travis.yml index d498d0305..ae25a6895 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,24 +90,24 @@ matrix: before_install: addons: <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-apache2 - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-leauto-upgrades - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" git: depth: false # This is needed to have the history to checkout old versions of certbot-auto. <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-certonly-standalone - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-sdists - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" diff --git a/tests/letstest/README.md b/tests/letstest/README.md index f8a15208e..5bd326e2a 100644 --- a/tests/letstest/README.md +++ b/tests/letstest/README.md @@ -15,12 +15,12 @@ Simple AWS testfarm scripts for certbot client testing are needed, they need to be requested via online webform. ## Installation and configuration -These tests require Python 2.7, awscli, boto3, PyYAML, and fabric<2.0. If you -have Python 2.7 and virtualenv installed, you can use requirements.txt to -create a virtual environment with a known set of dependencies by running: +These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you +have Python 3 installed, you can use requirements.txt to create a virtual +environment with a known set of dependencies by running: ``` -virtualenv --python $(command -v python2.7 || command -v python2 || command -v python) venv -. ./venv/bin/activate +python3 -m venv venv3 +. ./venv3/bin/activate pip install --requirement requirements.txt ``` diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 9ea9fe76b..09821e7dd 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -40,23 +40,16 @@ import socket import sys import time import traceback -import urllib2 import boto3 from botocore.exceptions import ClientError +from six.moves.urllib import error as urllib_error +from six.moves.urllib import request as urllib_request import yaml -import fabric -from fabric.api import cd -from fabric.api import env -from fabric.api import execute -from fabric.api import lcd -from fabric.api import local -from fabric.api import run -from fabric.api import sudo -from fabric.context_managers import shell_env -from fabric.operations import get -from fabric.operations import put +from fabric import Config +from fabric import Connection + # Command line parser #------------------------------------------------------------------------------- @@ -203,11 +196,11 @@ def block_until_http_ready(urlstring, wait_time=10, timeout=240): try: sys.stdout.write('.') sys.stdout.flush() - req = urllib2.Request(urlstring) - response = urllib2.urlopen(req) + req = urllib_request.Request(urlstring) + response = urllib_request.urlopen(req) #if response.code == 200: server_ready = True - except urllib2.URLError: + except urllib_error.URLError: pass time.sleep(wait_time) t_elapsed += wait_time @@ -244,76 +237,85 @@ def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20 # Fabric Routines #------------------------------------------------------------------------------- -def local_git_clone(repo_url): +def local_git_clone(local_cxn, repo_url): "clones master of repo_url" - with lcd(LOGDIR): - local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s letsencrypt'% repo_url) - local('tar czf le.tar.gz letsencrypt') + local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR) + local_cxn.local('cd %s && git clone %s letsencrypt'% (LOGDIR, repo_url)) + local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt'% LOGDIR) -def local_git_branch(repo_url, branch_name): +def local_git_branch(local_cxn, repo_url, branch_name): "clones branch of repo_url" - with lcd(LOGDIR): - local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s letsencrypt --branch %s --single-branch'%(repo_url, branch_name)) - local('tar czf le.tar.gz letsencrypt') + local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR) + local_cxn.local('cd %s && git clone %s letsencrypt --branch %s --single-branch'% + (LOGDIR, repo_url, branch_name)) + local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt' % LOGDIR) -def local_git_PR(repo_url, PRnumstr, merge_master=True): +def local_git_PR(local_cxn, repo_url, PRnumstr, merge_master=True): "clones specified pull request from repo_url and optionally merges into master" - with lcd(LOGDIR): - local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s letsencrypt'% repo_url) - local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr) - local('cd letsencrypt && git checkout lePRtest') - if merge_master: - local('cd letsencrypt && git remote update origin') - local('cd letsencrypt && git merge origin/master -m "testmerge"') - local('tar czf le.tar.gz letsencrypt') + local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR) + local_cxn.local('cd %s && git clone %s letsencrypt' % (LOGDIR, repo_url)) + local_cxn.local('cd %s && cd letsencrypt && ' + 'git fetch origin pull/%s/head:lePRtest' % (LOGDIR, PRnumstr)) + local_cxn.local('cd %s && cd letsencrypt && git checkout lePRtest' % LOGDIR) + if merge_master: + local_cxn.local('cd %s && cd letsencrypt && git remote update origin' % LOGDIR) + local_cxn.local('cd %s && cd letsencrypt && ' + 'git merge origin/master -m "testmerge"' % LOGDIR) + local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt' % LOGDIR) -def local_repo_to_remote(): +def local_repo_to_remote(cxn): "copies local tarball of repo to remote" - with lcd(LOGDIR): - put(local_path='le.tar.gz', remote_path='') - run('tar xzf le.tar.gz') + filename = 'le.tar.gz' + local_path = os.path.join(LOGDIR, filename) + cxn.put(local=local_path, remote='') + cxn.run('tar xzf %s' % filename) -def local_repo_clean(): +def local_repo_clean(local_cxn): "delete tarball" - with lcd(LOGDIR): - local('rm le.tar.gz') + filename = 'le.tar.gz' + local_path = os.path.join(LOGDIR, filename) + local_cxn.local('rm %s' % local_path) -def deploy_script(scriptpath, *args): +def deploy_script(cxn, scriptpath, *args): "copies to remote and executes local script" - #with lcd('scripts'): - put(local_path=scriptpath, remote_path='', mirror_local_mode=True) + cxn.put(local=scriptpath, remote='', preserve_mode=True) scriptfile = os.path.split(scriptpath)[1] args_str = ' '.join(args) - run('./'+scriptfile+' '+args_str) + cxn.run('./'+scriptfile+' '+args_str) -def run_boulder(): - with cd('$GOPATH/src/github.com/letsencrypt/boulder'): - run('sudo docker-compose up -d') +def run_boulder(cxn): + boulder_path = '$GOPATH/src/github.com/letsencrypt/boulder' + cxn.run('cd %s && sudo docker-compose up -d' % boulder_path) -def config_and_launch_boulder(instance): - execute(deploy_script, 'scripts/boulder_config.sh') - execute(run_boulder) +def config_and_launch_boulder(cxn, instance): + # yes, we're hardcoding the gopath. it's a predetermined AMI. + with cxn.prefix('export GOPATH=/home/ubuntu/gopath'): + deploy_script(cxn, 'scripts/boulder_config.sh') + run_boulder(cxn) -def install_and_launch_certbot(instance, boulder_url, target): - execute(local_repo_to_remote) - with shell_env(BOULDER_URL=boulder_url, - PUBLIC_IP=instance.public_ip_address, - PRIVATE_IP=instance.private_ip_address, - PUBLIC_HOSTNAME=instance.public_dns_name, - PIP_EXTRA_INDEX_URL=cl_args.alt_pip, - OS_TYPE=target['type']): - execute(deploy_script, cl_args.test_script) +def install_and_launch_certbot(cxn, instance, boulder_url, target): + local_repo_to_remote(cxn) + # This needs to be like this, I promise. 1) The env argument to run doesn't work. + # See https://github.com/fabric/fabric/issues/1744. 2) prefix() sticks an && between + # the commands, so it needs to be exports rather than no &&s in between for the script subshell. + with cxn.prefix('export BOULDER_URL=%s && export PUBLIC_IP=%s && export PRIVATE_IP=%s && ' + 'export PUBLIC_HOSTNAME=%s && export PIP_EXTRA_INDEX_URL=%s && ' + 'export OS_TYPE=%s' % + (boulder_url, + instance.public_ip_address, + instance.private_ip_address, + instance.public_dns_name, + cl_args.alt_pip, + target['type'])): + deploy_script(cxn, cl_args.test_script) -def grab_certbot_log(): +def grab_certbot_log(cxn): "grabs letsencrypt.log via cat into logged stdout" - sudo('if [ -f /var/log/letsencrypt/letsencrypt.log ]; then \ - cat /var/log/letsencrypt/letsencrypt.log; else echo "[novarlog]"; fi') + cxn.sudo('/bin/bash -l -i -c \'if [ -f "/var/log/letsencrypt/letsencrypt.log" ]; then ' + + 'cat "/var/log/letsencrypt/letsencrypt.log"; else echo "[novarlog]"; fi\'') # fallback file if /var/log is unwriteable...? correct? - sudo('if [ -f ./certbot.log ]; then \ - cat ./certbot.log; else echo "[nolocallog]"; fi') + cxn.sudo('/bin/bash -l -i -c \'if [ -f ./certbot.log ]; then ' + + 'cat ./certbot.log; else echo "[nolocallog]"; fi\'') def create_client_instance(ec2_client, target, security_group_id, subnet_id): @@ -341,7 +343,7 @@ def create_client_instance(ec2_client, target, security_group_id, subnet_id): userdata=userdata) -def test_client_process(inqueue, outqueue, boulder_url): +def test_client_process(fab_config, inqueue, outqueue, boulder_url): cur_proc = mp.current_process() for inreq in iter(inqueue.get, SENTINEL): ii, instance_id, target = inreq @@ -358,30 +360,31 @@ def test_client_process(inqueue, outqueue, boulder_url): print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name'])) instance = block_until_instance_ready(instance) print("server %s at %s"%(instance, instance.public_ip_address)) - env.host_string = "%s@%s"%(target['user'], instance.public_ip_address) - print(env.host_string) + host_string = "%s@%s"%(target['user'], instance.public_ip_address) + print(host_string) - try: - install_and_launch_certbot(instance, boulder_url, target) - outqueue.put((ii, target, Status.PASS)) - print("%s - %s SUCCESS"%(target['ami'], target['name'])) - except: - outqueue.put((ii, target, Status.FAIL)) - print("%s - %s FAIL"%(target['ami'], target['name'])) - traceback.print_exc(file=sys.stdout) - pass + with Connection(host_string, config=fab_config) as cxn: + try: + install_and_launch_certbot(cxn, instance, boulder_url, target) + outqueue.put((ii, target, Status.PASS)) + print("%s - %s SUCCESS"%(target['ami'], target['name'])) + except: + outqueue.put((ii, target, Status.FAIL)) + print("%s - %s FAIL"%(target['ami'], target['name'])) + traceback.print_exc(file=sys.stdout) + pass - # append server certbot.log to each per-machine output log - print("\n\ncertbot.log\n" + "-"*80 + "\n") - try: - execute(grab_certbot_log) - except: - print("log fail\n") - traceback.print_exc(file=sys.stdout) - pass + # append server certbot.log to each per-machine output log + print("\n\ncertbot.log\n" + "-"*80 + "\n") + try: + grab_certbot_log(cxn) + except: + print("log fail\n") + traceback.print_exc(file=sys.stdout) + pass -def cleanup(cl_args, instances, targetlist): +def cleanup(cl_args, instances, targetlist, boulder_server): print('Logs in ', LOGDIR) # If lengths of instances and targetlist aren't equal, instances failed to # start before running tests so leaving instances running for debugging @@ -402,19 +405,25 @@ def cleanup(cl_args, instances, targetlist): def main(): # Fabric library controlled through global env parameters - env.key_filename = KEYFILE - env.shell = '/bin/bash -l -i -c' - env.connection_attempts = 5 - env.timeout = 10 - # replace default SystemExit thrown by fabric during trouble - class FabricException(Exception): - pass - env['abort_exception'] = FabricException + fab_config = Config(overrides={ + "connect_kwargs": { + "key_filename": [KEYFILE], # https://github.com/fabric/fabric/issues/2007 + }, + "run": { + "echo": True, + "pty": True, + }, + "timeouts": { + "connect": 10, + }, + }) + # no network connection, so don't worry about closing this one. + local_cxn = Connection('localhost', config=fab_config) # Set up local copy of git repo #------------------------------------------------------------------------------- print("Making local dir for test repo and logs: %s"%LOGDIR) - local('mkdir %s'%LOGDIR) + local_cxn.local('mkdir %s'%LOGDIR) # figure out what git object to test and locally create it in LOGDIR print("Making local git repo") @@ -422,14 +431,14 @@ def main(): if cl_args.pull_request != '~': print('Testing PR %s '%cl_args.pull_request, "MERGING into master" if cl_args.merge_master else "") - execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master) + local_git_PR(local_cxn, cl_args.repo, cl_args.pull_request, cl_args.merge_master) elif cl_args.branch != '~': print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo)) - execute(local_git_branch, cl_args.repo, cl_args.branch) + local_git_branch(local_cxn, cl_args.repo, cl_args.branch) else: print('Testing master of %s'%cl_args.repo) - execute(local_git_clone, cl_args.repo) - except FabricException: + local_git_clone(local_cxn, cl_args.repo) + except BaseException: print("FAIL: trouble with git repo") traceback.print_exc() exit() @@ -437,7 +446,7 @@ def main(): # Set up EC2 instances #------------------------------------------------------------------------------- - configdata = yaml.load(open(cl_args.config_file, 'r')) + configdata = yaml.safe_load(open(cl_args.config_file, 'r')) targetlist = configdata['targets'] print('Testing against these images: [%d total]'%len(targetlist)) for target in targetlist: @@ -511,15 +520,16 @@ def main(): print(" server %s"%boulder_server) - # env.host_string defines the ssh user and host for connection - env.host_string = "ubuntu@%s"%boulder_server.public_ip_address - print("Boulder Server at (SSH):", env.host_string) + # host_string defines the ssh user and host for connection + host_string = "ubuntu@%s"%boulder_server.public_ip_address + print("Boulder Server at (SSH):", host_string) if not boulder_preexists: print("Configuring and Launching Boulder") - config_and_launch_boulder(boulder_server) - # blocking often unnecessary, but cheap EC2 VMs can get very slow - block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, - wait_time=10, timeout=500) + with Connection(host_string, config=fab_config) as boulder_cxn: + config_and_launch_boulder(boulder_cxn, boulder_server) + # blocking often unnecessary, but cheap EC2 VMs can get very slow + block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, + wait_time=10, timeout=500) boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) @@ -545,7 +555,7 @@ def main(): # initiate process execution for i in range(num_processes): - p = mp.Process(target=test_client_process, args=(inqueue, outqueue, boulder_url)) + p = mp.Process(target=test_client_process, args=(fab_config, inqueue, outqueue, boulder_url)) jobs.append(p) p.daemon = True # kills subprocesses if parent is killed p.start() @@ -569,7 +579,7 @@ def main(): outqueue.put(SENTINEL) # clean up - execute(local_repo_clean) + local_repo_clean(local_cxn) # print and save summary results results_file = open(LOGDIR+'/results', 'w') @@ -594,10 +604,7 @@ def main(): sys.exit(1) finally: - cleanup(cl_args, instances, targetlist) - - # kill any connections - fabric.network.disconnect_all() + cleanup(cl_args, instances, targetlist, boulder_server) if __name__ == '__main__': diff --git a/tests/letstest/requirements.txt b/tests/letstest/requirements.txt index 24bd77331..840e3e5d5 100644 --- a/tests/letstest/requirements.txt +++ b/tests/letstest/requirements.txt @@ -5,9 +5,9 @@ cffi==1.14.0 cryptography==2.8 docutils==0.15.2 enum34==1.1.9 -Fabric==1.14.1 -futures==3.3.0 +Fabric==2.5.0 ipaddress==1.0.23 +Invoke==1.4.1 jmespath==0.9.5 paramiko==2.7.1 pycparser==2.19 From 8e4dc0a48c6108311cf68ca1c683d29b15d49c97 Mon Sep 17 00:00:00 2001 From: Karan Suthar Date: Mon, 13 Apr 2020 23:11:39 +0530 Subject: [PATCH 60/72] Minor bugfixes (#7891) * Fix dangerous default argument * Remove unused imports * Remove unnecessary comprehension * Use literal syntax to create data structure * Use literal syntax instead of function calls to create data structure Co-authored-by: deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com> --- acme/acme/client.py | 4 +- acme/docs/conf.py | 1 - acme/tests/errors_test.py | 2 +- .../certbot_apache/_internal/apache_util.py | 4 +- .../certbot_apache/_internal/assertions.py | 4 +- .../certbot_apache/_internal/augeasparser.py | 6 +- .../certbot_apache/_internal/configurator.py | 18 +++--- .../certbot_apache/_internal/display_ops.py | 4 +- .../certbot_apache/_internal/dualparser.py | 4 +- .../certbot_apache/_internal/parser.py | 6 +- .../_internal/parsernode_util.py | 2 +- certbot-apache/tests/augeasnode_test.py | 1 - certbot-apache/tests/centos6_test.py | 6 +- certbot-apache/tests/centos_test.py | 4 +- certbot-apache/tests/configurator_test.py | 42 ++++++------- certbot-apache/tests/display_ops_test.py | 4 +- certbot-apache/tests/gentoo_test.py | 6 +- certbot-apache/tests/obj_test.py | 16 ++--- certbot-apache/tests/util.py | 38 ++++++------ .../certbot_tests/context.py | 1 - .../certbot_integration_tests/utils/misc.py | 2 +- .../certbot_compatibility_test/test_driver.py | 2 +- .../certbot_nginx/_internal/configurator.py | 4 +- .../certbot_nginx/_internal/display_ops.py | 4 +- .../certbot_nginx/_internal/parser.py | 6 +- .../certbot_nginx/_internal/parser_obj.py | 4 +- certbot-nginx/tests/configurator_test.py | 14 ++--- certbot-nginx/tests/obj_test.py | 36 +++++------ certbot-nginx/tests/parser_obj_test.py | 2 +- certbot-nginx/tests/parser_test.py | 62 +++++++++---------- .../certbot/_internal/cli/cli_constants.py | 14 ++--- certbot/certbot/_internal/main.py | 2 +- certbot/certbot/_internal/plugins/manual.py | 2 +- .../certbot/_internal/plugins/selection.py | 2 +- certbot/certbot/display/ops.py | 2 +- certbot/certbot/plugins/storage.py | 4 +- certbot/tests/display/ops_test.py | 8 +-- certbot/tests/display/util_test.py | 10 +-- certbot/tests/error_handler_test.py | 2 +- certbot/tests/hook_test.py | 2 +- certbot/tests/main_test.py | 4 +- certbot/tests/plugins/common_test.py | 12 ++-- certbot/tests/plugins/standalone_test.py | 6 +- certbot/tests/reverter_test.py | 8 +-- .../rebuild_dependencies.py | 2 +- tools/_venv_common.py | 4 +- tools/deactivate.py | 1 - windows-installer/construct.py | 4 +- 48 files changed, 197 insertions(+), 201 deletions(-) mode change 100755 => 100644 letsencrypt-auto-source/rebuild_dependencies.py diff --git a/acme/acme/client.py b/acme/acme/client.py index cbe543f91..3ce321ac9 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1123,8 +1123,8 @@ class ClientNetwork(object): debug_content = response.content.decode("utf-8") logger.debug('Received response:\nHTTP %d\n%s\n\n%s', response.status_code, - "\n".join(["{0}: {1}".format(k, v) - for k, v in response.headers.items()]), + "\n".join("{0}: {1}".format(k, v) + for k, v in response.headers.items()), debug_content) return response diff --git a/acme/docs/conf.py b/acme/docs/conf.py index a9c69d538..ba1a3aa8b 100644 --- a/acme/docs/conf.py +++ b/acme/docs/conf.py @@ -13,7 +13,6 @@ # serve to show the default. import os -import shlex import sys here = os.path.abspath(os.path.dirname(__file__)) diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index 1e5f3d479..c9c6f484f 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -35,7 +35,7 @@ class PollErrorTest(unittest.TestCase): def setUp(self): from acme.errors import PollError self.timeout = PollError( - exhausted=set([mock.sentinel.AR]), + exhausted={mock.sentinel.AR}, updated={}) self.invalid = PollError(exhausted=set(), updated={ mock.sentinel.AR: mock.sentinel.AR2}) diff --git a/certbot-apache/certbot_apache/_internal/apache_util.py b/certbot-apache/certbot_apache/_internal/apache_util.py index ebc8a26c9..862685027 100644 --- a/certbot-apache/certbot_apache/_internal/apache_util.py +++ b/certbot-apache/certbot_apache/_internal/apache_util.py @@ -130,7 +130,7 @@ def included_in_paths(filepath, paths): :rtype: bool """ - return any([fnmatch.fnmatch(filepath, path) for path in paths]) + return any(fnmatch.fnmatch(filepath, path) for path in paths) def parse_defines(apachectl): @@ -144,7 +144,7 @@ def parse_defines(apachectl): :rtype: dict """ - variables = dict() + variables = {} define_cmd = [apachectl, "-t", "-D", "DUMP_RUN_CFG"] matches = parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)") diff --git a/certbot-apache/certbot_apache/_internal/assertions.py b/certbot-apache/certbot_apache/_internal/assertions.py index e1b4cdcc8..2b2ce4f50 100644 --- a/certbot-apache/certbot_apache/_internal/assertions.py +++ b/certbot-apache/certbot_apache/_internal/assertions.py @@ -132,9 +132,9 @@ def assertEqualPathsList(first, second): # pragma: no cover Checks that the two lists of file paths match. This assertion allows for wildcard paths. """ - if any([isPass(path) for path in first]): + if any(isPass(path) for path in first): return - if any([isPass(path) for path in second]): + if any(isPass(path) for path in second): return for fpath in first: assert any([fnmatch.fnmatch(fpath, spath) for spath in second]) diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index f85d80923..3b2ce40d8 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -339,7 +339,7 @@ class AugeasBlockNode(AugeasDirectiveNode): def find_blocks(self, name, exclude=True): """Recursive search of BlockNodes from the sequence of children""" - nodes = list() + nodes = [] paths = self._aug_find_blocks(name) if exclude: paths = self.parser.exclude_dirs(paths) @@ -351,7 +351,7 @@ class AugeasBlockNode(AugeasDirectiveNode): def find_directives(self, name, exclude=True): """Recursive search of DirectiveNodes from the sequence of children""" - nodes = list() + nodes = [] ownpath = self.metadata.get("augeaspath") directives = self.parser.find_dir(name, start=ownpath, exclude=exclude) @@ -374,7 +374,7 @@ class AugeasBlockNode(AugeasDirectiveNode): :param str comment: Comment content to search for. """ - nodes = list() + nodes = [] ownpath = self.metadata.get("augeaspath") comments = self.parser.find_comments(comment, start=ownpath) diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 86d579954..dbfc15468 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -208,12 +208,12 @@ class ApacheConfigurator(common.Installer): super(ApacheConfigurator, self).__init__(*args, **kwargs) # Add name_server association dict - self.assoc = dict() # type: Dict[str, obj.VirtualHost] + self.assoc = {} # type: Dict[str, obj.VirtualHost] # Outstanding challenges self._chall_out = set() # type: Set[KeyAuthorizationAnnotatedChallenge] # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = dict() # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] # Temporary state for AutoHSTS enhancement @@ -436,7 +436,7 @@ class ApacheConfigurator(common.Installer): """Initializes the ParserNode parser root instance.""" if HAS_APACHECONFIG: - apache_vars = dict() + apache_vars = {} apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) @@ -548,7 +548,7 @@ class ApacheConfigurator(common.Installer): # Go through the vhosts, making sure that we cover all the names # present, but preferring the SSL vhosts - filtered_vhosts = dict() + filtered_vhosts = {} for vhost in vhosts: for name in vhost.get_names(): if vhost.ssl: @@ -574,7 +574,7 @@ class ApacheConfigurator(common.Installer): # Make sure we create SSL vhosts for the ones that are HTTP only # if requested. - return_vhosts = list() + return_vhosts = [] for vhost in dialog_output: if not vhost.ssl: return_vhosts.append(self.make_vhost_ssl(vhost)) @@ -1579,7 +1579,7 @@ class ApacheConfigurator(common.Installer): result.append(comment) sift = True - result.append('\n'.join(['# ' + l for l in chunk])) + result.append('\n'.join('# ' + l for l in chunk)) else: result.append('\n'.join(chunk)) return result, sift @@ -1719,7 +1719,7 @@ class ApacheConfigurator(common.Installer): for addr in vhost.addrs: # In Apache 2.2, when a NameVirtualHost directive is not # set, "*" and "_default_" will conflict when sharing a port - addrs = set((addr,)) + addrs = {addr,} if addr.get_addr() in ("*", "_default_"): addrs.update(obj.Addr((a, addr.get_port(),)) for a in ("*", "_default_")) @@ -1910,7 +1910,7 @@ class ApacheConfigurator(common.Installer): try: self._autohsts = self.storage.fetch("autohsts") except KeyError: - self._autohsts = dict() + self._autohsts = {} def _autohsts_save_state(self): """ @@ -2471,7 +2471,7 @@ class ApacheConfigurator(common.Installer): if len(matches) != 1: raise errors.PluginError("Unable to find Apache version") - return tuple([int(i) for i in matches[0].split(".")]) + return tuple(int(i) for i in matches[0].split(".")) def more_info(self): """Human-readable string to help understand the module""" diff --git a/certbot-apache/certbot_apache/_internal/display_ops.py b/certbot-apache/certbot_apache/_internal/display_ops.py index 1ae32bb47..dabf20606 100644 --- a/certbot-apache/certbot_apache/_internal/display_ops.py +++ b/certbot-apache/certbot_apache/_internal/display_ops.py @@ -21,7 +21,7 @@ def select_vhost_multiple(vhosts): :rtype: :class:`list`of type `~obj.Vhost` """ if not vhosts: - return list() + return [] tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] # Remove the extra newline from the last entry if tags_list: @@ -37,7 +37,7 @@ def select_vhost_multiple(vhosts): def _reversemap_vhosts(names, vhosts): """Helper function for select_vhost_multiple for mapping string representations back to actual vhost objects""" - return_vhosts = list() + return_vhosts = [] for selection in names: for vhost in vhosts: diff --git a/certbot-apache/certbot_apache/_internal/dualparser.py b/certbot-apache/certbot_apache/_internal/dualparser.py index aa66cf84c..eef8f2a0e 100644 --- a/certbot-apache/certbot_apache/_internal/dualparser.py +++ b/certbot-apache/certbot_apache/_internal/dualparser.py @@ -49,7 +49,7 @@ class DualNodeBase(object): pass_primary = assertions.isPassNodeList(primary_res) pass_secondary = assertions.isPassNodeList(secondary_res) - new_nodes = list() + new_nodes = [] if pass_primary and pass_secondary: # Both unimplemented @@ -221,7 +221,7 @@ class DualBlockNode(DualNodeBase): implementations to a list of tuples. """ - matched = list() + matched = [] for p in primary_list: match = None for s in secondary_list: diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 82d7df6aa..f6aa3fe48 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -30,7 +30,7 @@ class ApacheParser(object): """ arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") - fnmatch_chars = set(["*", "?", "\\", "[", "]"]) + fnmatch_chars = {"*", "?", "\\", "[", "]"} def __init__(self, root, vhostroot=None, version=(2, 4), configurator=None): @@ -945,8 +945,8 @@ def case_i(string): :param str string: string to make case i regex """ - return "".join(["[" + c.upper() + c.lower() + "]" - if c.isalpha() else c for c in re.escape(string)]) + return "".join("[" + c.upper() + c.lower() + "]" + if c.isalpha() else c for c in re.escape(string)) def get_aug_path(file_path): diff --git a/certbot-apache/certbot_apache/_internal/parsernode_util.py b/certbot-apache/certbot_apache/_internal/parsernode_util.py index d9646862a..0e39ec365 100644 --- a/certbot-apache/certbot_apache/_internal/parsernode_util.py +++ b/certbot-apache/certbot_apache/_internal/parsernode_util.py @@ -11,7 +11,7 @@ def validate_kwargs(kwargs, required_names): :param list required_names: List of required parameter names. """ - validated_kwargs = dict() + validated_kwargs = {} for name in required_names: try: validated_kwargs[name] = kwargs.pop(name) diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 4468a838e..270cc8c44 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -2,7 +2,6 @@ import mock import os -import unittest import util from certbot import errors diff --git a/certbot-apache/tests/centos6_test.py b/certbot-apache/tests/centos6_test.py index 15d086600..27b4f8e80 100644 --- a/certbot-apache/tests/centos6_test.py +++ b/certbot-apache/tests/centos6_test.py @@ -19,12 +19,12 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "test.example.com.conf"), os.path.join(aug_pre, "test.example.com.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "test.example.com"), obj.VirtualHost( os.path.join(prefix, "ssl.conf"), os.path.join(aug_pre, "ssl.conf/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), + {obj.Addr.fromstring("_default_:443")}, True, True, None) ] return vh_truth @@ -104,7 +104,7 @@ class CentOS6Tests(util.ApacheTest): pre_loadmods = self.config.parser.find_dir( "LoadModule", "ssl_module", exclude=False) # LoadModules are not within IfModule blocks - self.assertFalse(any(["ifmodule" in m.lower() for m in pre_loadmods])) + self.assertFalse(any("ifmodule" in m.lower() for m in pre_loadmods)) self.config.assoc["test.example.com"] = self.vh_truth[0] self.config.deploy_cert( "random.demo", "example/cert.pem", "example/key.pem", diff --git a/certbot-apache/tests/centos_test.py b/certbot-apache/tests/centos_test.py index 5e334c51e..40d1361d4 100644 --- a/certbot-apache/tests/centos_test.py +++ b/certbot-apache/tests/centos_test.py @@ -21,12 +21,12 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "centos.example.com.conf"), os.path.join(aug_pre, "centos.example.com.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "centos.example.com"), obj.VirtualHost( os.path.join(prefix, "ssl.conf"), os.path.join(aug_pre, "ssl.conf/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), + {obj.Addr.fromstring("_default_:443")}, True, True, None) ] return vh_truth diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 54f4c9251..694c1c0bb 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -99,7 +99,7 @@ class MultipleVhostsTest(util.ApacheTest): parserargs = ["server_root", "enmod", "dismod", "le_vhost_ext", "vhost_root", "logs_root", "challenge_location", "handle_modules", "handle_sites", "ctl"] - exp = dict() + exp = {} for k in ApacheConfigurator.OS_DEFAULTS: if k in parserargs: @@ -140,11 +140,9 @@ class MultipleVhostsTest(util.ApacheTest): mock_utility = mock_getutility() mock_utility.notification = mock.MagicMock(return_value=True) names = self.config.get_all_names() - self.assertEqual(names, set( - ["certbot.demo", "ocspvhost.com", "encryption-example.demo", + self.assertEqual(names, {"certbot.demo", "ocspvhost.com", "encryption-example.demo", "nonsym.link", "vhost.in.rootconf", "www.certbot.demo", - "duplicate.example.com"] - )) + "duplicate.example.com"}) @certbot_util.patch_get_utility() @mock.patch("certbot_apache._internal.configurator.socket.gethostbyaddr") @@ -154,9 +152,9 @@ class MultipleVhostsTest(util.ApacheTest): mock_utility.notification.return_value = True vhost = obj.VirtualHost( "fp", "ap", - set([obj.Addr(("8.8.8.8", "443")), + {obj.Addr(("8.8.8.8", "443")), obj.Addr(("zombo.com",)), - obj.Addr(("192.168.1.2"))]), + obj.Addr(("192.168.1.2"))}, True, False) self.config.vhosts.append(vhost) @@ -185,7 +183,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_bad_servername_alias(self): ssl_vh1 = obj.VirtualHost( - "fp1", "ap1", set([obj.Addr(("*", "443"))]), + "fp1", "ap1", {obj.Addr(("*", "443"))}, True, False) # pylint: disable=protected-access self.config._add_servernames(ssl_vh1) @@ -198,7 +196,7 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.config._add_servernames(self.vh_truth[2]) self.assertEqual( - self.vh_truth[2].get_names(), set(["*.le.co", "ip-172-30-0-17"])) + self.vh_truth[2].get_names(), {"*.le.co", "ip-172-30-0-17"}) def test_get_virtual_hosts(self): """Make sure all vhosts are being properly found.""" @@ -269,7 +267,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_choose_vhost_select_vhost_conflicting_non_ssl(self, mock_select): mock_select.return_value = self.vh_truth[3] conflicting_vhost = obj.VirtualHost( - "path", "aug_path", set([obj.Addr.fromstring("*:443")]), + "path", "aug_path", {obj.Addr.fromstring("*:443")}, True, True) self.config.vhosts.append(conflicting_vhost) @@ -278,14 +276,14 @@ class MultipleVhostsTest(util.ApacheTest): def test_find_best_http_vhost_default(self): vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr.fromstring("_default_:80")]), False, True) + "fp", "ap", {obj.Addr.fromstring("_default_:80")}, False, True) self.config.vhosts = [vh] self.assertEqual(self.config.find_best_http_vhost("foo.bar", False), vh) def test_find_best_http_vhost_port(self): port = "8080" vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr.fromstring("*:" + port)]), + "fp", "ap", {obj.Addr.fromstring("*:" + port)}, False, True, "encryption-example.demo") self.config.vhosts.append(vh) self.assertEqual(self.config.find_best_http_vhost("foo.bar", False, port), vh) @@ -313,8 +311,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_find_best_vhost_variety(self): # pylint: disable=protected-access ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), - obj.Addr(("zombo.com",))]), + "fp", "ap", {obj.Addr(("*", "443")), + obj.Addr(("zombo.com",))}, True, False) self.config.vhosts.append(ssl_vh) self.assertEqual(self.config._find_best_vhost("zombo.com"), ssl_vh) @@ -685,7 +683,7 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(ssl_vhost.path, "/files" + ssl_vhost.filep + "/IfModule/Virtualhost") self.assertEqual(len(ssl_vhost.addrs), 1) - self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs) + self.assertEqual({obj.Addr.fromstring("*:443")}, ssl_vhost.addrs) self.assertEqual(ssl_vhost.name, "encryption-example.demo") self.assertTrue(ssl_vhost.ssl) self.assertFalse(ssl_vhost.enabled) @@ -910,7 +908,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.parser.modules["rewrite_module"] = None mock_exe.return_value = True ssl_vh1 = obj.VirtualHost( - "fp1", "ap1", set([obj.Addr(("*", "443"))]), + "fp1", "ap1", {obj.Addr(("*", "443"))}, True, False) ssl_vh1.name = "satoshi.com" self.config.vhosts.append(ssl_vh1) @@ -1011,7 +1009,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_get_http_vhost_third_filter(self): ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443"))]), + "fp", "ap", {obj.Addr(("*", "443"))}, True, False) ssl_vh.name = "satoshi.com" self.config.vhosts.append(ssl_vh) @@ -1214,8 +1212,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_with_conflict(self): self.config.parser.modules["rewrite_module"] = None ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), - obj.Addr(("zombo.com",))]), + "fp", "ap", {obj.Addr(("*", "443")), + obj.Addr(("zombo.com",))}, True, False) # No names ^ this guy should conflict. @@ -1257,7 +1255,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.get_version = mock.Mock(return_value=(2, 3, 9)) # For full testing... give names... self.vh_truth[1].name = "default.com" - self.vh_truth[1].aliases = set(["yes.default.com"]) + self.vh_truth[1].aliases = {"yes.default.com"} # pylint: disable=protected-access self.config._enable_redirect(self.vh_truth[1], "") @@ -1268,7 +1266,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.get_version = mock.Mock(return_value=(2, 2)) # For full testing... give names... self.vh_truth[1].name = "default.com" - self.vh_truth[1].aliases = set(["yes.default.com"]) + self.vh_truth[1].aliases = {"yes.default.com"} # pylint: disable=protected-access self.config._enable_redirect(self.vh_truth[1], "") @@ -1609,7 +1607,7 @@ class MultiVhostsTest(util.ApacheTest): self.assertEqual(ssl_vhost.path, "/files" + ssl_vhost.filep + "/IfModule/VirtualHost") self.assertEqual(len(ssl_vhost.addrs), 1) - self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs) + self.assertEqual({obj.Addr.fromstring("*:443")}, ssl_vhost.addrs) self.assertEqual(ssl_vhost.name, "banana.vomit.com") self.assertTrue(ssl_vhost.ssl) self.assertFalse(ssl_vhost.enabled) diff --git a/certbot-apache/tests/display_ops_test.py b/certbot-apache/tests/display_ops_test.py index 50bdc03cf..e9f54a562 100644 --- a/certbot-apache/tests/display_ops_test.py +++ b/certbot-apache/tests/display_ops_test.py @@ -93,9 +93,9 @@ class SelectVhostTest(unittest.TestCase): self.vhosts.append( obj.VirtualHost( - "path", "aug_path", set([obj.Addr.fromstring("*:80")]), + "path", "aug_path", {obj.Addr.fromstring("*:80")}, False, False, - "wildcard.com", set(["*.wildcard.com"]))) + "wildcard.com", {"*.wildcard.com"})) self.assertEqual(self.vhosts[5], self._call(self.vhosts)) diff --git a/certbot-apache/tests/gentoo_test.py b/certbot-apache/tests/gentoo_test.py index 3b35dde54..5a35c17ae 100644 --- a/certbot-apache/tests/gentoo_test.py +++ b/certbot-apache/tests/gentoo_test.py @@ -21,19 +21,19 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "gentoo.example.com.conf"), os.path.join(aug_pre, "gentoo.example.com.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "gentoo.example.com"), obj.VirtualHost( os.path.join(prefix, "00_default_vhost.conf"), os.path.join(aug_pre, "00_default_vhost.conf/IfDefine/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "localhost"), obj.VirtualHost( os.path.join(prefix, "00_default_ssl_vhost.conf"), os.path.join(aug_pre, "00_default_ssl_vhost.conf" + "/IfDefine/IfDefine/IfModule/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), + {obj.Addr.fromstring("_default_:443")}, True, True, "localhost") ] return vh_truth diff --git a/certbot-apache/tests/obj_test.py b/certbot-apache/tests/obj_test.py index 1761b9c94..eac6a64ef 100644 --- a/certbot-apache/tests/obj_test.py +++ b/certbot-apache/tests/obj_test.py @@ -14,13 +14,13 @@ class VirtualHostTest(unittest.TestCase): self.addr_default = Addr.fromstring("_default_:443") self.vhost1 = VirtualHost( - "filep", "vh_path", set([self.addr1]), False, False, "localhost") + "filep", "vh_path", {self.addr1}, False, False, "localhost") self.vhost1b = VirtualHost( - "filep", "vh_path", set([self.addr1]), False, False, "localhost") + "filep", "vh_path", {self.addr1}, False, False, "localhost") self.vhost2 = VirtualHost( - "fp", "vhp", set([self.addr2]), False, False, "localhost") + "fp", "vhp", {self.addr2}, False, False, "localhost") def test_repr(self): self.assertEqual(repr(self.addr2), @@ -42,7 +42,7 @@ class VirtualHostTest(unittest.TestCase): complex_vh = VirtualHost( "fp", "vhp", - set([Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")]), + {Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")}, False, False) self.assertTrue(complex_vh.conflicts([self.addr1])) self.assertTrue(complex_vh.conflicts([self.addr2])) @@ -57,14 +57,14 @@ class VirtualHostTest(unittest.TestCase): def test_same_server(self): from certbot_apache._internal.obj import VirtualHost no_name1 = VirtualHost( - "fp", "vhp", set([self.addr1]), False, False, None) + "fp", "vhp", {self.addr1}, False, False, None) no_name2 = VirtualHost( - "fp", "vhp", set([self.addr2]), False, False, None) + "fp", "vhp", {self.addr2}, False, False, None) no_name3 = VirtualHost( - "fp", "vhp", set([self.addr_default]), + "fp", "vhp", {self.addr_default}, False, False, None) no_name4 = VirtualHost( - "fp", "vhp", set([self.addr2, self.addr_default]), + "fp", "vhp", {self.addr2, self.addr_default}, False, False, None) self.assertTrue(self.vhost1.same_server(self.vhost2)) diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index 4994103f3..a39e5225d 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -142,71 +142,71 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "encryption-example.conf"), os.path.join(aug_pre, "encryption-example.conf/Virtualhost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "encryption-example.demo"), obj.VirtualHost( os.path.join(prefix, "default-ssl.conf"), os.path.join(aug_pre, "default-ssl.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), True, True), + {obj.Addr.fromstring("_default_:443")}, True, True), obj.VirtualHost( os.path.join(prefix, "000-default.conf"), os.path.join(aug_pre, "000-default.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80"), - obj.Addr.fromstring("[::]:80")]), + {obj.Addr.fromstring("*:80"), + obj.Addr.fromstring("[::]:80")}, False, True, "ip-172-30-0-17"), obj.VirtualHost( os.path.join(prefix, "certbot.conf"), os.path.join(aug_pre, "certbot.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "certbot.demo", aliases=["www.certbot.demo"]), obj.VirtualHost( os.path.join(prefix, "mod_macro-example.conf"), os.path.join(aug_pre, "mod_macro-example.conf/Macro/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, modmacro=True), obj.VirtualHost( os.path.join(prefix, "default-ssl-port-only.conf"), os.path.join(aug_pre, ("default-ssl-port-only.conf/" "IfModule/VirtualHost")), - set([obj.Addr.fromstring("_default_:443")]), True, True), + {obj.Addr.fromstring("_default_:443")}, True, True), obj.VirtualHost( os.path.join(prefix, "wildcard.conf"), os.path.join(aug_pre, "wildcard.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "ip-172-30-0-17", aliases=["*.blue.purple.com"]), obj.VirtualHost( os.path.join(prefix, "ocsp-ssl.conf"), os.path.join(aug_pre, "ocsp-ssl.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("10.2.3.4:443")]), True, True, + {obj.Addr.fromstring("10.2.3.4:443")}, True, True, "ocspvhost.com"), obj.VirtualHost( os.path.join(prefix, "non-symlink.conf"), os.path.join(aug_pre, "non-symlink.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "nonsym.link"), obj.VirtualHost( os.path.join(prefix, "default-ssl-port-only.conf"), os.path.join(aug_pre, "default-ssl-port-only.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), True, True, ""), + {obj.Addr.fromstring("*:80")}, True, True, ""), obj.VirtualHost( os.path.join(temp_dir, config_name, "apache2/apache2.conf"), "/files" + os.path.join(temp_dir, config_name, "apache2/apache2.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "vhost.in.rootconf"), obj.VirtualHost( os.path.join(prefix, "duplicatehttp.conf"), os.path.join(aug_pre, "duplicatehttp.conf/VirtualHost"), - set([obj.Addr.fromstring("10.2.3.4:80")]), False, True, + {obj.Addr.fromstring("10.2.3.4:80")}, False, True, "duplicate.example.com"), obj.VirtualHost( os.path.join(prefix, "duplicatehttps.conf"), os.path.join(aug_pre, "duplicatehttps.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("10.2.3.4:443")]), True, True, + {obj.Addr.fromstring("10.2.3.4:443")}, True, True, "duplicate.example.com")] return vh_truth if config_name == "debian_apache_2_4/multi_vhosts": @@ -217,27 +217,27 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "default.conf"), os.path.join(aug_pre, "default.conf/VirtualHost[1]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "ip-172-30-0-17"), obj.VirtualHost( os.path.join(prefix, "default.conf"), os.path.join(aug_pre, "default.conf/VirtualHost[2]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "banana.vomit.com"), obj.VirtualHost( os.path.join(prefix, "multi-vhost.conf"), os.path.join(aug_pre, "multi-vhost.conf/VirtualHost[1]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "1.multi.vhost.tld"), obj.VirtualHost( os.path.join(prefix, "multi-vhost.conf"), os.path.join(aug_pre, "multi-vhost.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "2.multi.vhost.tld"), obj.VirtualHost( os.path.join(prefix, "multi-vhost.conf"), os.path.join(aug_pre, "multi-vhost.conf/VirtualHost[2]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "3.multi.vhost.tld")] return vh_truth return None # pragma: no cover diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index 6f8670000..e295aefd7 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -1,5 +1,4 @@ """Module to handle the context of integration tests.""" -import logging import os import shutil import sys diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index b08f11e89..fbb965034 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -236,7 +236,7 @@ def generate_csr(domains, key_path, csr_path, key_type=RSA_KEY_TYPE): file_h.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, key)) req = crypto.X509Req() - san = ', '.join(['DNS:{0}'.format(item) for item in domains]) + san = ', '.join('DNS:{0}'.format(item) for item in domains) san_constraint = crypto.X509Extension(b'subjectAltName', False, san.encode('utf-8')) req.add_extensions([san_constraint]) diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index d719f583b..5140dc8ea 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -96,7 +96,7 @@ def test_authenticator(plugin, config, temp_dir): def _create_achalls(plugin): """Returns a list of annotated challenges to test on plugin""" - achalls = list() + achalls = [] names = plugin.get_testable_domain_names() for domain in names: prefs = plugin.get_chall_pref(domain) diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index ddab48512..a903c12bf 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -748,7 +748,7 @@ class NginxConfigurator(common.Installer): # if there is no separate SSL block, break the block into two and # choose the SSL block. - if vhost.ssl and any([not addr.ssl for addr in vhost.addrs]): + if vhost.ssl and any(not addr.ssl for addr in vhost.addrs): _, vhost = self._split_block(vhost) header_directives = [ @@ -983,7 +983,7 @@ class NginxConfigurator(common.Installer): logger.warning("NGINX derivative %s is not officially supported by" " certbot", product_name) - nginx_version = tuple([int(i) for i in product_version.split(".")]) + nginx_version = tuple(int(i) for i in product_version.split(".")) # nginx < 0.8.48 uses machine hostname as default server_name instead of # the empty string diff --git a/certbot-nginx/certbot_nginx/_internal/display_ops.py b/certbot-nginx/certbot_nginx/_internal/display_ops.py index bbb47f98a..356cc506c 100644 --- a/certbot-nginx/certbot_nginx/_internal/display_ops.py +++ b/certbot-nginx/certbot_nginx/_internal/display_ops.py @@ -17,7 +17,7 @@ def select_vhost_multiple(vhosts): :rtype: :class:`list`of type `~obj.Vhost` """ if not vhosts: - return list() + return [] tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] # Remove the extra newline from the last entry if tags_list: @@ -33,7 +33,7 @@ def select_vhost_multiple(vhosts): def _reversemap_vhosts(names, vhosts): """Helper function for select_vhost_multiple for mapping string representations back to actual vhost objects""" - return_vhosts = list() + return_vhosts = [] for selection in names: for vhost in vhosts: diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 0c1151826..bb0bb7d6f 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -404,9 +404,9 @@ class NginxParser(object): if directive and directive[0] == 'listen': # Exclude one-time use parameters which will cause an error if repeated. # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen - exclude = set(('default_server', 'default', 'setfib', 'fastopen', 'backlog', + exclude = {'default_server', 'default', 'setfib', 'fastopen', 'backlog', 'rcvbuf', 'sndbuf', 'accept_filter', 'deferred', 'bind', - 'ipv6only', 'reuseport', 'so_keepalive')) + 'ipv6only', 'reuseport', 'so_keepalive'} for param in exclude: # See: github.com/certbot/certbot/pull/6223#pullrequestreview-143019225 @@ -578,7 +578,7 @@ def _update_or_add_directives(directives, insert_at_top, block): INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite', 'add_header']) +REPEATABLE_DIRECTIVES = {'server_name', 'listen', INCLUDE, 'rewrite', 'add_header'} COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 61b31b2d5..390e18e4d 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -206,7 +206,7 @@ class Sentence(Parsable): :returns: whether this lists is parseable by `Sentence`. """ return isinstance(lists, list) and len(lists) > 0 and \ - all([isinstance(elem, six.string_types) for elem in lists]) + all(isinstance(elem, six.string_types) for elem in lists) def parse(self, raw_list, add_spaces=False): """ Parses a list of string types into this object. @@ -214,7 +214,7 @@ class Sentence(Parsable): if add_spaces: raw_list = _space_list(raw_list) if not isinstance(raw_list, list) or \ - any([not isinstance(elem, six.string_types) for elem in raw_list]): + any(not isinstance(elem, six.string_types) for elem in raw_list): raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") self._data = raw_list diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 0d9d6d356..0a04a22a4 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -104,7 +104,7 @@ class NginxConfiguratorTest(util.NginxTest): filep = self.config.parser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(filep, None, None, None, - set(['.example.com', 'example.*']), + {'.example.com', 'example.*'}, None, [0]) self.config.parser.add_server_directives( mock_vhost, @@ -150,11 +150,11 @@ class NginxConfiguratorTest(util.NginxTest): self._test_choose_vhosts_common('ipv6.com', 'ipv6_conf') def _test_choose_vhosts_common(self, name, conf): - conf_names = {'localhost_conf': set(['localhost', r'~^(www\.)?(example|bar)\.']), - 'server_conf': set(['somename', 'another.alias', 'alias']), - 'example_conf': set(['.example.com', 'example.*']), - 'foo_conf': set(['*.www.foo.com', '*.www.example.com']), - 'ipv6_conf': set(['ipv6.com'])} + conf_names = {'localhost_conf': {'localhost', r'~^(www\.)?(example|bar)\.'}, + 'server_conf': {'somename', 'another.alias', 'alias'}, + 'example_conf': {'.example.com', 'example.*'}, + 'foo_conf': {'*.www.foo.com', '*.www.example.com'}, + 'ipv6_conf': {'ipv6.com'}} conf_path = {'localhost': "etc_nginx/nginx.conf", 'alias': "etc_nginx/nginx.conf", @@ -177,7 +177,7 @@ class NginxConfiguratorTest(util.NginxTest): self.assertTrue(vhost.ipv6_enabled()) # Make sure that we have SSL enabled also for IPv6 addr self.assertTrue( - any([True for x in vhost.addrs if x.ssl and x.ipv6])) + any(True for x in vhost.addrs if x.ssl and x.ipv6)) def test_choose_vhosts_bad(self): bad_results = ['www.foo.com', 'example', 't.www.bar.co', diff --git a/certbot-nginx/tests/obj_test.py b/certbot-nginx/tests/obj_test.py index db808229f..93b9493eb 100644 --- a/certbot-nginx/tests/obj_test.py +++ b/certbot-nginx/tests/obj_test.py @@ -98,10 +98,10 @@ class AddrTest(unittest.TestCase): def test_set_inclusion(self): from certbot_nginx._internal.obj import Addr - set_a = set([self.addr1, self.addr2]) + set_a = {self.addr1, self.addr2} addr1b = Addr.fromstring("192.168.1.1") addr2b = Addr.fromstring("192.168.1.1:* ssl") - set_b = set([addr1b, addr2b]) + set_b = {addr1b, addr2b} self.assertEqual(set_a, set_b) @@ -120,8 +120,8 @@ class VirtualHostTest(unittest.TestCase): ] self.vhost1 = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw1, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw1, []) raw2 = [ ['listen', '69.50.225.155:9000'], [['if', '($scheme', '!=', '"https") '], @@ -130,24 +130,24 @@ class VirtualHostTest(unittest.TestCase): ] self.vhost2 = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw2, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw2, []) raw3 = [ ['listen', '69.50.225.155:9000'], ['rewrite', '^(.*)$', '$scheme://www.domain.com$1', 'permanent'] ] self.vhost3 = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw3, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw3, []) raw4 = [ ['listen', '69.50.225.155:9000'], ['server_name', 'return.com'] ] self.vhost4 = VirtualHost( "filp", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw4, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw4, []) raw_has_hsts = [ ['listen', '69.50.225.155:9000'], ['server_name', 'return.com'], @@ -155,16 +155,16 @@ class VirtualHostTest(unittest.TestCase): ] self.vhost_has_hsts = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw_has_hsts, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw_has_hsts, []) def test_eq(self): from certbot_nginx._internal.obj import Addr from certbot_nginx._internal.obj import VirtualHost vhost1b = VirtualHost( "filep", - set([Addr.fromstring("localhost blah")]), False, False, - set(['localhost']), [], []) + {Addr.fromstring("localhost blah")}, False, False, + {'localhost'}, [], []) self.assertEqual(vhost1b, self.vhost1) self.assertEqual(str(vhost1b), str(self.vhost1)) @@ -203,8 +203,8 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], []] vhost_haystack = VirtualHost( "filp", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), test_haystack, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, test_haystack, []) test_bad_haystack = [['listen', '80'], ['root', '/var/www/html'], ['index', 'index.html index.htm index.nginx-debian.html'], ['server_name', 'two.functorkitten.xyz'], ['listen', '443 ssl'], @@ -219,8 +219,8 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], []] vhost_bad_haystack = VirtualHost( "filp", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), test_bad_haystack, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, test_bad_haystack, []) self.assertTrue(vhost_haystack.contains_list(test_needle)) self.assertFalse(vhost_bad_haystack.contains_list(test_needle)) diff --git a/certbot-nginx/tests/parser_obj_test.py b/certbot-nginx/tests/parser_obj_test.py index 132f83771..bb7834701 100644 --- a/certbot-nginx/tests/parser_obj_test.py +++ b/certbot-nginx/tests/parser_obj_test.py @@ -80,7 +80,7 @@ class ParsingHooksTest(unittest.TestCase): fake_parser1.should_parse = lambda x: False parsing_hooks.return_value = (fake_parser1,) self.assertRaises(errors.MisconfigurationError, parse_raw, []) - parsing_hooks.return_value = tuple() + parsing_hooks.return_value = () self.assertRaises(errors.MisconfigurationError, parse_raw, []) def test_parse_raw_passes_add_spaces(self): diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 72cfc0716..21dd1043d 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -126,7 +126,7 @@ class NginxParserTest(util.NginxTest): vhost = obj.VirtualHost(nparser.abs_path('sites-enabled/globalssl.com'), [obj.Addr('4.8.2.6', '57', True, False, False, False)], - True, True, set(['globalssl.com']), [], [0]) + True, True, {'globalssl.com'}, [], [0]) globalssl_com = [x for x in vhosts if 'globalssl.com' in x.filep][0] self.assertEqual(vhost, globalssl_com) @@ -139,8 +139,8 @@ class NginxParserTest(util.NginxTest): [obj.Addr('', '8080', False, False, False, False)], False, True, - set(['localhost', - r'~^(www\.)?(example|bar)\.']), + {'localhost', + r'~^(www\.)?(example|bar)\.'}, [], [10, 1, 9]) vhost2 = obj.VirtualHost(nparser.abs_path('nginx.conf'), [obj.Addr('somename', '8080', False, False, @@ -148,7 +148,7 @@ class NginxParserTest(util.NginxTest): obj.Addr('', '8000', False, False, False, False)], False, True, - set(['somename', 'another.alias', 'alias']), + {'somename', 'another.alias', 'alias'}, [], [10, 1, 12]) vhost3 = obj.VirtualHost(nparser.abs_path('sites-enabled/example.com'), [obj.Addr('69.50.225.155', '9000', @@ -156,19 +156,19 @@ class NginxParserTest(util.NginxTest): obj.Addr('127.0.0.1', '', False, False, False, False)], False, True, - set(['.example.com', 'example.*']), [], [0]) + {'.example.com', 'example.*'}, [], [0]) vhost4 = obj.VirtualHost(nparser.abs_path('sites-enabled/default'), [obj.Addr('myhost', '', False, True, False, False), obj.Addr('otherhost', '', False, True, False, False)], - False, True, set(['www.example.org']), + False, True, {'www.example.org'}, [], [0]) vhost5 = obj.VirtualHost(nparser.abs_path('foo.conf'), [obj.Addr('*', '80', True, True, False, False)], - True, True, set(['*.www.foo.com', - '*.www.example.com']), + True, True, {'*.www.foo.com', + '*.www.example.com'}, [], [2, 1, 0]) self.assertEqual(14, len(vhosts)) @@ -208,11 +208,11 @@ class NginxParserTest(util.NginxTest): nparser = parser.NginxParser(self.config_path) mock_vhost = obj.VirtualHost(nparser.abs_path('nginx.conf'), None, None, None, - set(['localhost', - r'~^(www\.)?(example|bar)\.']), + {'localhost', + r'~^(www\.)?(example|bar)\.'}, None, [10, 1, 9]) example_com = nparser.abs_path('sites-enabled/example.com') - names = set(['.example.com', 'example.*']) + names = {'.example.com', 'example.*'} mock_vhost.filep = example_com mock_vhost.names = names mock_vhost.path = [0] @@ -232,8 +232,8 @@ class NginxParserTest(util.NginxTest): nparser = parser.NginxParser(self.config_path) mock_vhost = obj.VirtualHost(nparser.abs_path('nginx.conf'), None, None, None, - set(['localhost', - r'~^(www\.)?(example|bar)\.']), + {'localhost', + r'~^(www\.)?(example|bar)\.'}, None, [10, 1, 9]) nparser.add_server_directives(mock_vhost, [['foo', 'bar'], ['\n ', 'ssl_certificate', ' ', @@ -243,7 +243,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual(1, len(re.findall(ssl_re, dump))) example_com = nparser.abs_path('sites-enabled/example.com') - names = set(['.example.com', 'example.*']) + names = {'.example.com', 'example.*'} mock_vhost.filep = example_com mock_vhost.names = names mock_vhost.path = [0] @@ -264,7 +264,7 @@ class NginxParserTest(util.NginxTest): ]]]) server_conf = nparser.abs_path('server.conf') - names = set(['alias', 'another.alias', 'somename']) + names = {'alias', 'another.alias', 'somename'} mock_vhost.filep = server_conf mock_vhost.names = names mock_vhost.path = [] @@ -279,7 +279,7 @@ class NginxParserTest(util.NginxTest): example_com = nparser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(example_com, None, None, None, - set(['.example.com', 'example.*']), + {'.example.com', 'example.*'}, None, [0]) nparser.add_server_directives(mock_vhost, [['\n ', '#', ' ', 'what a nice comment']]) @@ -301,7 +301,7 @@ class NginxParserTest(util.NginxTest): def test_replace_server_directives(self): nparser = parser.NginxParser(self.config_path) - target = set(['.example.com', 'example.*']) + target = {'.example.com', 'example.*'} filep = nparser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(filep, None, None, None, target, None, [0]) nparser.update_or_add_server_directives( @@ -314,7 +314,7 @@ class NginxParserTest(util.NginxTest): ['server_name', 'foobar.com'], ['#', COMMENT], ['server_name', 'example.*'], [] ]]]) - mock_vhost.names = set(['foobar.com', 'example.*']) + mock_vhost.names = {'foobar.com', 'example.*'} nparser.update_or_add_server_directives( mock_vhost, [['ssl_certificate', 'cert.pem']]) self.assertEqual( @@ -328,19 +328,19 @@ class NginxParserTest(util.NginxTest): def test_get_best_match(self): target_name = 'www.eff.org' - names = [set(['www.eff.org', 'irrelevant.long.name.eff.org', '*.org']), - set(['eff.org', 'ww2.eff.org', 'test.www.eff.org']), - set(['*.eff.org', '.www.eff.org']), - set(['.eff.org', '*.org']), - set(['www.eff.', 'www.eff.*', '*.www.eff.org']), - set(['example.com', r'~^(www\.)?(eff.+)', '*.eff.*']), - set(['*', r'~^(www\.)?(eff.+)']), - set(['www.*', r'~^(www\.)?(eff.+)', '.test.eff.org']), - set(['*.org', r'*.eff.org', 'www.eff.*']), - set(['*.www.eff.org', 'www.*']), - set(['*.org']), - set([]), - set(['example.com'])] + names = [{'www.eff.org', 'irrelevant.long.name.eff.org', '*.org'}, + {'eff.org', 'ww2.eff.org', 'test.www.eff.org'}, + {'*.eff.org', '.www.eff.org'}, + {'.eff.org', '*.org'}, + {'www.eff.', 'www.eff.*', '*.www.eff.org'}, + {'example.com', r'~^(www\.)?(eff.+)', '*.eff.*'}, + {'*', r'~^(www\.)?(eff.+)'}, + {'www.*', r'~^(www\.)?(eff.+)', '.test.eff.org'}, + {'*.org', r'*.eff.org', 'www.eff.*'}, + {'*.www.eff.org', 'www.*'}, + {'*.org'}, + set(), + {'example.com'}] winners = [('exact', 'www.eff.org'), (None, None), ('exact', '.www.eff.org'), diff --git a/certbot/certbot/_internal/cli/cli_constants.py b/certbot/certbot/_internal/cli/cli_constants.py index 748ae0d94..4bc84bfe7 100644 --- a/certbot/certbot/_internal/cli/cli_constants.py +++ b/certbot/certbot/_internal/cli/cli_constants.py @@ -91,17 +91,17 @@ ARGPARSE_PARAMS_TO_REMOVE = ("const", "nargs", "type",) # These sets are used when to help detect options set by the user. -EXIT_ACTIONS = set(("help", "version",)) +EXIT_ACTIONS = {"help", "version",} -ZERO_ARG_ACTIONS = set(("store_const", "store_true", - "store_false", "append_const", "count",)) +ZERO_ARG_ACTIONS = {"store_const", "store_true", + "store_false", "append_const", "count",} # Maps a config option to a set of config options that may have modified it. # This dictionary is used recursively, so if A modifies B and B modifies C, # it is determined that C was modified by the user if A was modified. -VAR_MODIFIERS = {"account": set(("server",)), - "renew_hook": set(("deploy_hook",)), - "server": set(("dry_run", "staging",)), - "webroot_map": set(("webroot_path",))} +VAR_MODIFIERS = {"account": {"server",}, + "renew_hook": {"deploy_hook",}, + "server": {"dry_run", "staging",}, + "webroot_map": {"webroot_path",}} diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 94ccb8b5e..264f9667e 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -890,7 +890,7 @@ def enhance(config, plugins): """ supported_enhancements = ["hsts", "redirect", "uir", "staple"] # Check that at least one enhancement was requested on command line - oldstyle_enh = any([getattr(config, enh) for enh in supported_enhancements]) + oldstyle_enh = any(getattr(config, enh) for enh in supported_enhancements) if not enhancements.are_requested(config) and not oldstyle_enh: msg = ("Please specify one or more enhancement types to configure. To list " "the available enhancement types, run:\n\n%s --help enhance\n") diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index 87ccdbd7e..b46622796 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -71,7 +71,7 @@ permitted by DNS standards.) super(Authenticator, self).__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() - self.env = dict() \ + self.env = {} \ # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] self.subsequent_dns_challenge = False self.subsequent_any_challenge = False diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index 6d87e4b07..53cef3969 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -138,7 +138,7 @@ def choose_plugin(prepared, question): while True: disp = z_util(interfaces.IDisplay) - if "CERTBOT_AUTO" in os.environ and names == set(("apache", "nginx")): + if "CERTBOT_AUTO" in os.environ and names == {"apache", "nginx"}: # The possibility of being offered exactly apache and nginx here # is new interactivity brought by https://github.com/certbot/certbot/issues/4079, # so set apache as a default for those kinds of non-interactive use diff --git a/certbot/certbot/display/ops.py b/certbot/certbot/display/ops.py index f24f6ed99..21d169a55 100644 --- a/certbot/certbot/display/ops.py +++ b/certbot/certbot/display/ops.py @@ -195,7 +195,7 @@ def _choose_names_manually(prompt_prefix=""): cli_flag="--domains", force_interactive=True) if code == display_util.OK: - invalid_domains = dict() + invalid_domains = {} retry_message = "" try: domain_list = display_util.separate_list_input(input_) diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index fe3a97dd5..9123087e7 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -42,7 +42,7 @@ class PluginStorage(object): :raises .errors.PluginStorageError: when unable to open or read the file """ - data = dict() # type: Dict[str, Any] + data = {} # type: Dict[str, Any] filedata = "" try: with open(self._storagepath, 'r') as fh: @@ -107,7 +107,7 @@ class PluginStorage(object): self._initialize_storage() if not self._classkey in self._data.keys(): - self._data[self._classkey] = dict() + self._data[self._classkey] = {} self._data[self._classkey][key] = value def fetch(self, key): diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 5df7bfcf8..327f1bcbe 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -271,7 +271,7 @@ class ChooseNamesTest(unittest.TestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_valid_return(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = (display_util.OK, ["example.com"]) names = self._call(self.mock_install) @@ -280,7 +280,7 @@ class ChooseNamesTest(unittest.TestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_namees_override_question(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = (display_util.OK, ["example.com"]) names = self._call(self.mock_install, "Custom") self.assertEqual(names, ["example.com"]) @@ -289,14 +289,14 @@ class ChooseNamesTest(unittest.TestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_nothing_selected(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = (display_util.OK, []) self.assertEqual(self._call(self.mock_install), []) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_cancel(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = ( display_util.CANCEL, ["example.com"]) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 615f33406..eccfee7db 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -173,14 +173,14 @@ class FileOutputDisplayTest(unittest.TestCase): code, tag_list = self.displayer.checklist( "msg", TAGS, force_interactive=True) self.assertEqual( - (code, set(tag_list)), (display_util.OK, set(["tag1", "tag2"]))) + (code, set(tag_list)), (display_util.OK, {"tag1", "tag2"})) @mock.patch("certbot.display.util.input_with_timeout") def test_checklist_empty(self, mock_input): mock_input.return_value = "" code, tag_list = self.displayer.checklist("msg", TAGS, force_interactive=True) self.assertEqual( - (code, set(tag_list)), (display_util.OK, set(["tag1", "tag2", "tag3"]))) + (code, set(tag_list)), (display_util.OK, {"tag1", "tag2", "tag3"})) @mock.patch("certbot.display.util.input_with_timeout") def test_checklist_miss_valid(self, mock_input): @@ -212,9 +212,9 @@ class FileOutputDisplayTest(unittest.TestCase): ["2", "3"], ] exp = [ - set(["tag1"]), - set(["tag1", "tag2"]), - set(["tag2", "tag3"]), + {"tag1"}, + {"tag1", "tag2"}, + {"tag2", "tag3"}, ] for i, list_ in enumerate(indices): set_tags = set( diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 011313208..899dbc611 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -42,7 +42,7 @@ class ErrorHandlerTest(unittest.TestCase): from certbot._internal import error_handler self.init_func = mock.MagicMock() - self.init_args = set((42,)) + self.init_args = {42,} self.init_kwargs = {'foo': 'bar'} self.handler = error_handler.ErrorHandler(self.init_func, *self.init_args, diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 54d0fcf67..3b7a94489 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -24,7 +24,7 @@ class ValidateHooksTest(unittest.TestCase): self._call(config) types = [call[0][1] for call in mock_validate_hook.call_args_list] - self.assertEqual(set(("pre", "post", "deploy",)), set(types[:-1])) + self.assertEqual({"pre", "post", "deploy",}, set(types[:-1])) # This ensures error messages are about deploy hooks when appropriate self.assertEqual("renew", types[-1]) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 8b2645876..92afc3fef 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1583,9 +1583,9 @@ class EnhanceTest(test_util.ConfigTestCase): not_req_enh = ["uir"] self.assertTrue(mock_client.enhance_config.called) self.assertTrue( - all([getattr(mock_client.config, e) for e in req_enh])) + all(getattr(mock_client.config, e) for e in req_enh)) self.assertFalse( - any([getattr(mock_client.config, e) for e in not_req_enh])) + any(getattr(mock_client.config, e) for e in not_req_enh)) self.assertTrue( "example.com" in mock_client.enhance_config.call_args[0][0]) diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 7543f28f3..1f339a6f3 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -94,7 +94,7 @@ class InstallerTest(test_util.ConfigTestCase): self.reverter = self.installer.reverter def test_add_to_real_checkpoint(self): - files = set(("foo.bar", "baz.qux",)) + files = {"foo.bar", "baz.qux",} save_notes = "foo bar baz qux" self._test_wrapped_method("add_to_checkpoint", files, save_notes) @@ -105,7 +105,7 @@ class InstallerTest(test_util.ConfigTestCase): self._test_add_to_checkpoint_common(True) def _test_add_to_checkpoint_common(self, temporary): - files = set(("foo.bar", "baz.qux",)) + files = {"foo.bar", "baz.qux",} save_notes = "foo bar baz qux" installer_func = functools.partial(self.installer.add_to_checkpoint, @@ -242,17 +242,17 @@ class AddrTest(unittest.TestCase): def test_set_inclusion(self): from certbot.plugins.common import Addr - set_a = set([self.addr1, self.addr2]) + set_a = {self.addr1, self.addr2} addr1b = Addr.fromstring("192.168.1.1") addr2b = Addr.fromstring("192.168.1.1:*") - set_b = set([addr1b, addr2b]) + set_b = {addr1b, addr2b} self.assertEqual(set_a, set_b) - set_c = set([self.addr4, self.addr5]) + set_c = {self.addr4, self.addr5} addr4b = Addr.fromstring("[fe00::1]") addr5b = Addr.fromstring("[fe00::1]:*") - set_d = set([addr4b, addr5b]) + set_d = {addr4b, addr5b} self.assertEqual(set_c, set_d) diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 701abe109..862accb92 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -161,7 +161,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup(["chall1"]) self.assertEqual(self.auth.served, { - "server1": set(), "server2": set(["chall2", "chall3"])}) + "server1": set(), "server2": {"chall2", "chall3"}}) self.auth.servers.stop.assert_called_once_with(1) self.auth.servers.running.return_value = { @@ -169,12 +169,12 @@ class AuthenticatorTest(unittest.TestCase): } self.auth.cleanup(["chall2"]) self.assertEqual(self.auth.served, { - "server1": set(), "server2": set(["chall3"])}) + "server1": set(), "server2": {"chall3"}}) self.assertEqual(1, self.auth.servers.stop.call_count) self.auth.cleanup(["chall3"]) self.assertEqual(self.auth.served, { - "server1": set(), "server2": set([])}) + "server1": set(), "server2": set()}) self.auth.servers.stop.assert_called_with(2) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index fc9133c5f..e592bcbdc 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -87,7 +87,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): # Check to make sure new files are also checked... self.assertRaises(errors.ReverterError, self.reverter.add_to_checkpoint, - set([config3]), "invalid save") + {config3}, "invalid save") def test_multiple_saves_and_temp_revert(self): self.reverter.add_to_temp_checkpoint(self.sets[0], "save1") @@ -414,9 +414,9 @@ def setup_test_files(): with open(config2, "w") as file_fd: file_fd.write("directive-dir2") - sets = [set([config1]), - set([config2]), - set([config1, config2])] + sets = [{config1}, + {config2}, + {config1, config2}] return config1, config2, dir1, dir2, sets diff --git a/letsencrypt-auto-source/rebuild_dependencies.py b/letsencrypt-auto-source/rebuild_dependencies.py old mode 100755 new mode 100644 index 6d1ec15ff..3093b6bb0 --- a/letsencrypt-auto-source/rebuild_dependencies.py +++ b/letsencrypt-auto-source/rebuild_dependencies.py @@ -103,7 +103,7 @@ def _requirements_from_one_distribution(distribution, verbose): os.chmod(script, 0o755) _write_to(authoritative_constraints, '\n'.join( - ['{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items()])) + '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) command = ['docker', 'run', '--rm', '--cidfile', cid_file, '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 75d0d5d33..5196cf9c4 100644 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -124,7 +124,9 @@ def _check_version(version_str, major_version): return False -def subprocess_with_print(cmd, env=os.environ, shell=False): +def subprocess_with_print(cmd, env=None, shell=False): + if env is None: + env = os.environ print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) subprocess.check_call(cmd, env=env, shell=shell) diff --git a/tools/deactivate.py b/tools/deactivate.py index 10c9ecd35..214c0595c 100644 --- a/tools/deactivate.py +++ b/tools/deactivate.py @@ -17,7 +17,6 @@ import sys from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa import josepy as jose from acme import client as acme_client diff --git a/windows-installer/construct.py b/windows-installer/construct.py index f0724f5f4..4b05c926a 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -153,7 +153,7 @@ extra_preamble=pywin32_paths.py '''.format(certbot_version=certbot_version, installer_suffix='win_amd64' if PYTHON_BITNESS == 64 else 'win32', python_bitness=PYTHON_BITNESS, - python_version='.'.join([str(item) for item in PYTHON_VERSION]))) + python_version='.'.join(str(item) for item in PYTHON_VERSION))) return installer_cfg_path @@ -184,7 +184,7 @@ if __name__ == '__main__': if sys.version_info[:2] != PYTHON_VERSION[:2]: raise RuntimeError('This script must be run with Python {0}' - .format('.'.join([str(item) for item in PYTHON_VERSION[0:2]]))) + .format('.'.join(str(item) for item in PYTHON_VERSION[0:2]))) if struct.calcsize('P') * 8 != PYTHON_BITNESS: raise RuntimeError('This script must be run with a {0} bit version of Python.' From cd0acf5dcc94263706b08d7f45d741fbc07c9196 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 13 Apr 2020 14:32:22 -0700 Subject: [PATCH 61/72] Do not require mock in Python 3 in acme module (#7894) Part of #7886. This PR conditionally installs mock in acme/setup.py based on setuptools version and python version, when possible. It then updates acme tests to use unittest.mock when mock isn't available. * Conditionally install mock in acme * use unittest.mock when third-party mock isn't available in acme * error when trying to build wheels with old setuptools --- acme/setup.py | 12 +++++++++++- acme/tests/challenges_test.py | 5 ++++- acme/tests/client_test.py | 5 ++++- acme/tests/errors_test.py | 5 ++++- acme/tests/magic_typing_test.py | 5 ++++- acme/tests/messages_test.py | 5 ++++- acme/tests/standalone_test.py | 5 ++++- 7 files changed, 35 insertions(+), 7 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 0527b3fb5..356410efe 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -15,7 +17,6 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'mock', # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13.1', 'pyrfc3339', @@ -26,6 +27,15 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'pytest', 'pytest-xdist', diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index 433c7bb82..b6a3d1f83 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -3,7 +3,10 @@ import unittest import josepy as jose import OpenSSL -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import requests from six.moves.urllib import parse as urllib_parse diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 010974a32..444737e15 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -6,7 +6,10 @@ import json import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL import requests from six.moves import http_client # pylint: disable=import-error diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index c9c6f484f..660c122d3 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -1,7 +1,10 @@ """Tests for acme.errors.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock class BadNonceTest(unittest.TestCase): diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 60b4a5df4..43efe8eff 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock class MagicTypingTest(unittest.TestCase): diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index d36e2cc99..9c2955f7e 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -2,7 +2,10 @@ import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges import test_util diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 8face7c7b..7a0f5ddde 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -4,7 +4,10 @@ import threading import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import requests from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error From 77871ba71c05dead938d700fefcfd719646301a1 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 13 Apr 2020 14:33:23 -0700 Subject: [PATCH 62/72] Do not require mock in Python 3 in certbot module (#7895) Part of #7886. This PR conditionally installs mock in `certbot/setup.py` based on setuptools version and python version, when possible. It then updates `certbot` tests to use `unittest.mock` when `mock` isn't available. * Conditionally install mock in certbot * use unittest.mock when third-party mock isn't available in certbot * Add type:ignores because of https://github.com/python/mypy/issues/1153 * error when trying to build wheels with old setuptools --- certbot/certbot/plugins/dns_test_common.py | 5 ++++- certbot/certbot/plugins/dns_test_common_lexicon.py | 5 ++++- certbot/certbot/tests/util.py | 5 ++++- certbot/setup.py | 12 ++++++++++-- certbot/tests/account_test.py | 5 ++++- certbot/tests/auth_handler_test.py | 5 ++++- certbot/tests/cert_manager_test.py | 5 ++++- certbot/tests/cli_test.py | 5 ++++- certbot/tests/client_test.py | 5 ++++- certbot/tests/compat/filesystem_test.py | 5 ++++- certbot/tests/configuration_test.py | 5 ++++- certbot/tests/crypto_util_test.py | 5 ++++- certbot/tests/display/completer_test.py | 5 ++++- certbot/tests/display/ops_test.py | 5 ++++- certbot/tests/display/util_test.py | 5 ++++- certbot/tests/eff_test.py | 5 ++++- certbot/tests/error_handler_test.py | 5 ++++- certbot/tests/errors_test.py | 5 ++++- certbot/tests/hook_test.py | 5 ++++- certbot/tests/lock_test.py | 5 ++++- certbot/tests/log_test.py | 5 ++++- certbot/tests/main_test.py | 5 ++++- certbot/tests/ocsp_test.py | 5 ++++- certbot/tests/plugins/common_test.py | 5 ++++- certbot/tests/plugins/disco_test.py | 5 ++++- certbot/tests/plugins/dns_common_lexicon_test.py | 5 ++++- certbot/tests/plugins/dns_common_test.py | 5 ++++- certbot/tests/plugins/enhancements_test.py | 5 ++++- certbot/tests/plugins/manual_test.py | 5 ++++- certbot/tests/plugins/null_test.py | 5 ++++- certbot/tests/plugins/selection_test.py | 5 ++++- certbot/tests/plugins/standalone_test.py | 5 ++++- certbot/tests/plugins/storage_test.py | 5 ++++- certbot/tests/plugins/util_test.py | 5 ++++- certbot/tests/plugins/webroot_test.py | 5 ++++- certbot/tests/renewal_test.py | 5 ++++- certbot/tests/renewupdater_test.py | 5 ++++- certbot/tests/reporter_test.py | 5 ++++- certbot/tests/reverter_test.py | 5 ++++- certbot/tests/storage_test.py | 5 ++++- certbot/tests/util_test.py | 5 ++++- 41 files changed, 170 insertions(+), 42 deletions(-) diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index 9ef76c2c3..d5044d336 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -2,7 +2,10 @@ import configobj import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six from acme import challenges diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index c77d6da9e..1bef06042 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -1,7 +1,10 @@ """Base test class for DNS authenticators built on Lexicon.""" import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 8b28b1080..92f52a852 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -10,7 +10,10 @@ import unittest from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import pkg_resources import six diff --git a/certbot/setup.py b/certbot/setup.py index 514aec8c5..143e1a10a 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -47,7 +47,6 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', @@ -62,7 +61,8 @@ install_requires = [ # So this dependency is not added for old Linux distributions with old setuptools, # in order to allow these systems to build certbot from sources. pywin32_req = 'pywin32>=227' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py -if StrictVersion(setuptools_version) >= StrictVersion('36.2'): +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: install_requires.append(pywin32_req + " ; sys_platform == 'win32'") elif 'bdist_wheel' in sys.argv[1:]: raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' @@ -73,6 +73,14 @@ elif os.name == 'nt': # setuptools, pywin32 will not be specified as a dependency. install_requires.append(pywin32_req) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'coverage', 'ipdb', diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 4a6ed3e01..6c6e6c860 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -4,7 +4,10 @@ import json import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz from acme import messages diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 7ab3a2baa..6cd207b4b 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -3,7 +3,10 @@ import functools import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from acme import challenges diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index bea64f09c..d956fd04f 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -7,7 +7,10 @@ import tempfile import unittest import configobj -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot._internal import configuration diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 7d21f8bb8..592c40be7 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -4,7 +4,10 @@ import copy import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 3e0e5b212..cbc058c7a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -5,7 +5,10 @@ import tempfile import unittest from josepy import interfaces -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index fdfb1ffe9..1c2d2df0d 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -3,7 +3,10 @@ import contextlib import errno import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import util from certbot._internal import lock diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index d748b9bfb..1f8a0e803 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.configuration.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot._internal import constants diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index d52e3acdb..481d83894 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -2,7 +2,10 @@ import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL import zope.component diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index a183fd14f..0852ab175 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -7,7 +7,10 @@ import string import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from six.moves import reload_module # pylint: disable=import-error from certbot.compat import filesystem # pylint: disable=ungrouped-imports diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 327f1bcbe..a683e1d3d 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,7 +4,10 @@ import sys import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from acme import messages diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index eccfee7db..3e492e9ab 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -4,7 +4,10 @@ import socket import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from certbot import errors diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index cdd7908a3..c4a25da69 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.eff.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import requests from certbot._internal import constants diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 899dbc611..e5a95c3a8 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -4,7 +4,10 @@ import signal import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.compat import os diff --git a/certbot/tests/errors_test.py b/certbot/tests/errors_test.py index d6c829322..792868df0 100644 --- a/certbot/tests/errors_test.py +++ b/certbot/tests/errors_test.py @@ -1,7 +1,10 @@ """Tests for certbot.errors.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import messages from certbot import achallenges diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 3b7a94489..32081f9d0 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.hooks.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 5a48009fd..2f887d33e 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -3,7 +3,10 @@ import functools import multiprocessing import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot.compat import os diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 5b0918ce5..5cd287c2e 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -5,7 +5,10 @@ import sys import time import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import messages diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 92afc3fef..8113b2bc4 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -13,7 +13,10 @@ import traceback import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 9eb49e115..af54844cf 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -10,7 +10,10 @@ from cryptography.exceptions import InvalidSignature from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes # type: ignore -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz from certbot import errors diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 1f339a6f3..344e6312f 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -4,7 +4,10 @@ import shutil import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges from certbot import achallenges diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index eec0795e3..5a0a392b0 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -3,7 +3,10 @@ import functools import string import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pkg_resources import six import zope.interface diff --git a/certbot/tests/plugins/dns_common_lexicon_test.py b/certbot/tests/plugins/dns_common_lexicon_test.py index 986362ca9..a67430f3e 100644 --- a/certbot/tests/plugins/dns_common_lexicon_test.py +++ b/certbot/tests/plugins/dns_common_lexicon_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.plugins import dns_common_lexicon from certbot.plugins import dns_test_common_lexicon diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index eba3c89d6..993f3b461 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -4,7 +4,10 @@ import collections import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/plugins/enhancements_test.py b/certbot/tests/plugins/enhancements_test.py index 05fbc5028..a20a6864f 100644 --- a/certbot/tests/plugins/enhancements_test.py +++ b/certbot/tests/plugins/enhancements_test.py @@ -1,7 +1,10 @@ """Tests for new style enhancements""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot._internal.plugins import null from certbot.plugins import enhancements diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 6cdef148a..933c759d6 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import challenges diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index db0213813..47708e340 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.plugins.null.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index c66473ad1..e5e6db031 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from certbot import errors diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 862accb92..751b9d943 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -5,7 +5,10 @@ from socket import errno as socket_errors # type: ignore import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL.crypto # pylint: disable=unused-import import six diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index e9ca2007f..4b0d1da83 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -2,7 +2,10 @@ import json import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot.compat import filesystem diff --git a/certbot/tests/plugins/util_test.py b/certbot/tests/plugins/util_test.py index c41e55222..9387b4ae7 100644 --- a/certbot/tests/plugins/util_test.py +++ b/certbot/tests/plugins/util_test.py @@ -1,7 +1,10 @@ """Tests for certbot.plugins.util.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.compat import os diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index fade12bb1..e57e09eae 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -10,7 +10,10 @@ import tempfile import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import challenges diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index e92211ea2..1fc54b42e 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.renewal""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges from certbot import errors diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index c6f8f3713..b5ecddb5a 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -1,7 +1,10 @@ """Tests for renewal updater interfaces""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import interfaces from certbot._internal import main diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 3d7c80172..7d03f1821 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index e592bcbdc..d67aa431a 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -5,7 +5,10 @@ import shutil import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from certbot import errors diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 0f7620b78..5aa37824d 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -6,7 +6,10 @@ import stat import unittest import configobj -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz import six diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 3ff09a83f..6dd0f964c 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -4,7 +4,10 @@ import errno import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from six.moves import reload_module # pylint: disable=import-error From ff732bf97538fe1ab94589d4ddeff294026f3ddd Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 13 Apr 2020 17:09:24 -0700 Subject: [PATCH 63/72] Revert the last two mock PRs (#7903) * Revert "Do not require mock in Python 3 in certbot module (#7895)" This reverts commit 77871ba71c05dead938d700fefcfd719646301a1. * Revert "Do not require mock in Python 3 in acme module (#7894)" This reverts commit cd0acf5dcc94263706b08d7f45d741fbc07c9196. --- acme/setup.py | 12 +----------- acme/tests/challenges_test.py | 5 +---- acme/tests/client_test.py | 5 +---- acme/tests/errors_test.py | 5 +---- acme/tests/magic_typing_test.py | 5 +---- acme/tests/messages_test.py | 5 +---- acme/tests/standalone_test.py | 5 +---- certbot/certbot/plugins/dns_test_common.py | 5 +---- certbot/certbot/plugins/dns_test_common_lexicon.py | 5 +---- certbot/certbot/tests/util.py | 5 +---- certbot/setup.py | 12 ++---------- certbot/tests/account_test.py | 5 +---- certbot/tests/auth_handler_test.py | 5 +---- certbot/tests/cert_manager_test.py | 5 +---- certbot/tests/cli_test.py | 5 +---- certbot/tests/client_test.py | 5 +---- certbot/tests/compat/filesystem_test.py | 5 +---- certbot/tests/configuration_test.py | 5 +---- certbot/tests/crypto_util_test.py | 5 +---- certbot/tests/display/completer_test.py | 5 +---- certbot/tests/display/ops_test.py | 5 +---- certbot/tests/display/util_test.py | 5 +---- certbot/tests/eff_test.py | 5 +---- certbot/tests/error_handler_test.py | 5 +---- certbot/tests/errors_test.py | 5 +---- certbot/tests/hook_test.py | 5 +---- certbot/tests/lock_test.py | 5 +---- certbot/tests/log_test.py | 5 +---- certbot/tests/main_test.py | 5 +---- certbot/tests/ocsp_test.py | 5 +---- certbot/tests/plugins/common_test.py | 5 +---- certbot/tests/plugins/disco_test.py | 5 +---- certbot/tests/plugins/dns_common_lexicon_test.py | 5 +---- certbot/tests/plugins/dns_common_test.py | 5 +---- certbot/tests/plugins/enhancements_test.py | 5 +---- certbot/tests/plugins/manual_test.py | 5 +---- certbot/tests/plugins/null_test.py | 5 +---- certbot/tests/plugins/selection_test.py | 5 +---- certbot/tests/plugins/standalone_test.py | 5 +---- certbot/tests/plugins/storage_test.py | 5 +---- certbot/tests/plugins/util_test.py | 5 +---- certbot/tests/plugins/webroot_test.py | 5 +---- certbot/tests/renewal_test.py | 5 +---- certbot/tests/renewupdater_test.py | 5 +---- certbot/tests/reporter_test.py | 5 +---- certbot/tests/reverter_test.py | 5 +---- certbot/tests/storage_test.py | 5 +---- certbot/tests/util_test.py | 5 +---- 48 files changed, 49 insertions(+), 205 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 356410efe..0527b3fb5 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,7 +1,5 @@ -from distutils.version import StrictVersion import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -17,6 +15,7 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', + 'mock', # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13.1', 'pyrfc3339', @@ -27,15 +26,6 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] -setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - dev_extras = [ 'pytest', 'pytest-xdist', diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index b6a3d1f83..433c7bb82 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -3,10 +3,7 @@ import unittest import josepy as jose import OpenSSL -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import requests from six.moves.urllib import parse as urllib_parse diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 444737e15..010974a32 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -6,10 +6,7 @@ import json import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import OpenSSL import requests from six.moves import http_client # pylint: disable=import-error diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index 660c122d3..c9c6f484f 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -1,10 +1,7 @@ """Tests for acme.errors.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock class BadNonceTest(unittest.TestCase): diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 43efe8eff..60b4a5df4 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -2,10 +2,7 @@ import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock class MagicTypingTest(unittest.TestCase): diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index 9c2955f7e..d36e2cc99 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -2,10 +2,7 @@ import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from acme import challenges import test_util diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 7a0f5ddde..8face7c7b 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -4,10 +4,7 @@ import threading import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import requests from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index d5044d336..9ef76c2c3 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -2,10 +2,7 @@ import configobj import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore +import mock import six from acme import challenges diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index 1bef06042..c77d6da9e 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -1,10 +1,7 @@ """Base test class for DNS authenticators built on Lexicon.""" import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore +import mock from requests.exceptions import HTTPError from requests.exceptions import RequestException diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 92f52a852..8b28b1080 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -10,10 +10,7 @@ import unittest from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore +import mock import OpenSSL import pkg_resources import six diff --git a/certbot/setup.py b/certbot/setup.py index 143e1a10a..514aec8c5 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -47,6 +47,7 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', + 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', @@ -61,8 +62,7 @@ install_requires = [ # So this dependency is not added for old Linux distributions with old setuptools, # in order to allow these systems to build certbot from sources. pywin32_req = 'pywin32>=227' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py -setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) -if setuptools_known_environment_markers: +if StrictVersion(setuptools_version) >= StrictVersion('36.2'): install_requires.append(pywin32_req + " ; sys_platform == 'win32'") elif 'bdist_wheel' in sys.argv[1:]: raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' @@ -73,14 +73,6 @@ elif os.name == 'nt': # setuptools, pywin32 will not be specified as a dependency. install_requires.append(pywin32_req) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - dev_extras = [ 'coverage', 'ipdb', diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 6c6e6c860..4a6ed3e01 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -4,10 +4,7 @@ import json import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import pytz from acme import messages diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 6cd207b4b..7ab3a2baa 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -3,10 +3,7 @@ import functools import logging import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import zope.component from acme import challenges diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index d956fd04f..bea64f09c 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -7,10 +7,7 @@ import tempfile import unittest import configobj -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot._internal import configuration diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 592c40be7..7d21f8bb8 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -4,10 +4,7 @@ import copy import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index cbc058c7a..3e0e5b212 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -5,10 +5,7 @@ import tempfile import unittest from josepy import interfaces -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot import util diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index 1c2d2df0d..fdfb1ffe9 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -3,10 +3,7 @@ import contextlib import errno import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import util from certbot._internal import lock diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index 1f8a0e803..d748b9bfb 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -1,10 +1,7 @@ """Tests for certbot._internal.configuration.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot._internal import constants diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 481d83894..d52e3acdb 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -2,10 +2,7 @@ import logging import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import OpenSSL import zope.component diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 0852ab175..a183fd14f 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -7,10 +7,7 @@ import string import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from six.moves import reload_module # pylint: disable=import-error from certbot.compat import filesystem # pylint: disable=ungrouped-imports diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index a683e1d3d..327f1bcbe 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,10 +4,7 @@ import sys import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import zope.component from acme import messages diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 3e492e9ab..eccfee7db 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -4,10 +4,7 @@ import socket import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from certbot import errors diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index c4a25da69..cdd7908a3 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -1,10 +1,7 @@ """Tests for certbot._internal.eff.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import requests from certbot._internal import constants diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index e5a95c3a8..899dbc611 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -4,10 +4,7 @@ import signal import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot.compat import os diff --git a/certbot/tests/errors_test.py b/certbot/tests/errors_test.py index 792868df0..d6c829322 100644 --- a/certbot/tests/errors_test.py +++ b/certbot/tests/errors_test.py @@ -1,10 +1,7 @@ """Tests for certbot.errors.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from acme import messages from certbot import achallenges diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 32081f9d0..3b7a94489 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -1,10 +1,7 @@ """Tests for certbot._internal.hooks.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot import util diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 2f887d33e..5a48009fd 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -3,10 +3,7 @@ import functools import multiprocessing import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot.compat import os diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 5cd287c2e..5b0918ce5 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -5,10 +5,7 @@ import sys import time import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from acme import messages diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 8113b2bc4..92afc3fef 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -13,10 +13,7 @@ import traceback import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import pytz import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index af54844cf..9eb49e115 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -10,10 +10,7 @@ from cryptography.exceptions import InvalidSignature from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes # type: ignore -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import pytz from certbot import errors diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 344e6312f..1f339a6f3 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -4,10 +4,7 @@ import shutil import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from acme import challenges from certbot import achallenges diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index 5a0a392b0..eec0795e3 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -3,10 +3,7 @@ import functools import string import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import pkg_resources import six import zope.interface diff --git a/certbot/tests/plugins/dns_common_lexicon_test.py b/certbot/tests/plugins/dns_common_lexicon_test.py index a67430f3e..986362ca9 100644 --- a/certbot/tests/plugins/dns_common_lexicon_test.py +++ b/certbot/tests/plugins/dns_common_lexicon_test.py @@ -2,10 +2,7 @@ import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot.plugins import dns_common_lexicon from certbot.plugins import dns_test_common_lexicon diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index 993f3b461..eba3c89d6 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -4,10 +4,7 @@ import collections import logging import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot import util diff --git a/certbot/tests/plugins/enhancements_test.py b/certbot/tests/plugins/enhancements_test.py index a20a6864f..05fbc5028 100644 --- a/certbot/tests/plugins/enhancements_test.py +++ b/certbot/tests/plugins/enhancements_test.py @@ -1,10 +1,7 @@ """Tests for new style enhancements""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot._internal.plugins import null from certbot.plugins import enhancements diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 933c759d6..6cdef148a 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -2,10 +2,7 @@ import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from acme import challenges diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index 47708e340..db0213813 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -1,10 +1,7 @@ """Tests for certbot._internal.plugins.null.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index e5e6db031..c66473ad1 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -2,10 +2,7 @@ import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import zope.component from certbot import errors diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 751b9d943..862accb92 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -5,10 +5,7 @@ from socket import errno as socket_errors # type: ignore import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import OpenSSL.crypto # pylint: disable=unused-import import six diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index 4b0d1da83..e9ca2007f 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -2,10 +2,7 @@ import json import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot.compat import filesystem diff --git a/certbot/tests/plugins/util_test.py b/certbot/tests/plugins/util_test.py index 9387b4ae7..c41e55222 100644 --- a/certbot/tests/plugins/util_test.py +++ b/certbot/tests/plugins/util_test.py @@ -1,10 +1,7 @@ """Tests for certbot.plugins.util.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot.compat import os diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index e57e09eae..fade12bb1 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -10,10 +10,7 @@ import tempfile import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from acme import challenges diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index 1fc54b42e..e92211ea2 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -1,10 +1,7 @@ """Tests for certbot._internal.renewal""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from acme import challenges from certbot import errors diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index b5ecddb5a..c6f8f3713 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -1,10 +1,7 @@ """Tests for renewal updater interfaces""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import interfaces from certbot._internal import main diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 7d03f1821..3d7c80172 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -2,10 +2,7 @@ import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index d67aa431a..e592bcbdc 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -5,10 +5,7 @@ import shutil import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from certbot import errors diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 5aa37824d..0f7620b78 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -6,10 +6,7 @@ import stat import unittest import configobj -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import pytz import six diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 6dd0f964c..3ff09a83f 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -4,10 +4,7 @@ import errno import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from six.moves import reload_module # pylint: disable=import-error From 569df2d37a02f95096f0004369491ac24f8630e6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 14 Apr 2020 17:01:59 -0700 Subject: [PATCH 64/72] Remove Ubuntu 19.04 tests. (#7906) This PR fixes the Travis failures that can be seen https://travis-ci.com/certbot/certbot/builds/160258644. Running the tests locally, it looks like Ubuntu has started shutting down the 19.04 repos which makes sense as this release has been EOL'd. See https://wiki.ubuntu.com/Releases. I have the full suite including the test farm tests running at https://travis-ci.com/github/certbot/certbot/builds/160269969 with this change. The issue of adding 19.10 to our test farm tests is tracked by #7851. I think that issue is important and it's in our current milestone, but I'd personally rather get our tests passing for now and try to expand them to run on other systems later. --- tests/letstest/apache2_targets.yaml | 5 ----- tests/letstest/targets.yaml | 5 ----- 2 files changed, 10 deletions(-) diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml index 1450a8578..76415b27e 100644 --- a/tests/letstest/apache2_targets.yaml +++ b/tests/letstest/apache2_targets.yaml @@ -1,11 +1,6 @@ targets: #----------------------------------------------------------------------------- #Ubuntu - - ami: ami-08ab45c4343f5f5c6 - name: ubuntu19.04 - type: ubuntu - virt: hvm - user: ubuntu - ami: ami-095192256fe1477ad name: ubuntu18.04LTS type: ubuntu diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 188be8e24..06055a9a5 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -1,11 +1,6 @@ targets: #----------------------------------------------------------------------------- #Ubuntu - - ami: ami-08ab45c4343f5f5c6 - name: ubuntu19.04 - type: ubuntu - virt: hvm - user: ubuntu - ami: ami-095192256fe1477ad name: ubuntu18.04LTS type: ubuntu From 127d2dc307fe827695a8ec69a373687b67c75dae Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:27:55 -0700 Subject: [PATCH 65/72] Do not require mock in Python 3 in acme module (#7910) Part of #7886. This PR conditionally installs mock in `acme/setup.py` based on setuptools version and python version, when possible. It then updates `acme` tests to use `unittest.mock` when `mock` isn't available. Now with `type: ignore` as appropriate. Once the "future steps" of #7886 are finished, and mypy is on Python 3, the `pragma no cover`s and `type ignore`s will be gone. * Conditionally install mock in acme * error out on newer python and older setuptools * error when trying to build wheels with old setuptools * use unittest.mock when third-party mock isn't available in acme, with no cover and type ignore --- acme/setup.py | 12 +++++++++++- acme/tests/challenges_test.py | 5 ++++- acme/tests/client_test.py | 5 ++++- acme/tests/errors_test.py | 5 ++++- acme/tests/magic_typing_test.py | 5 ++++- acme/tests/messages_test.py | 5 ++++- acme/tests/standalone_test.py | 5 ++++- 7 files changed, 35 insertions(+), 7 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 0527b3fb5..356410efe 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -15,7 +17,6 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'mock', # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13.1', 'pyrfc3339', @@ -26,6 +27,15 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'pytest', 'pytest-xdist', diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index 433c7bb82..70371051c 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -3,7 +3,10 @@ import unittest import josepy as jose import OpenSSL -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import requests from six.moves.urllib import parse as urllib_parse diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 010974a32..c90cad9b0 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -6,7 +6,10 @@ import json import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import requests from six.moves import http_client # pylint: disable=import-error diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index c9c6f484f..fb90a3f0d 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -1,7 +1,10 @@ """Tests for acme.errors.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore class BadNonceTest(unittest.TestCase): diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 60b4a5df4..9e4fd29f5 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore class MagicTypingTest(unittest.TestCase): diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index d36e2cc99..890a5f413 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -2,7 +2,10 @@ import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from acme import challenges import test_util diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 8face7c7b..d03b56535 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -4,7 +4,10 @@ import threading import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import requests from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error From 35fb99b86f4802694acfdefe5bcc5ba1d28e70c6 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:28:47 -0700 Subject: [PATCH 66/72] Do not require mock in Python 3 in certbot module (#7911) This PR is exactly the same as #7895, but know we know a little bit more about what was going on with `mypy`. Part of #7886. This PR conditionally installs mock in `certbot/setup.py` based on setuptools version and python version, when possible. It then updates `certbot` tests to use `unittest.mock` when `mock` isn't available. * Conditionally install mock in certbot * use unittest.mock when third-party mock isn't available in certbot * Add type:ignores because of https://github.com/python/mypy/issues/1153 * error out on newer python and older setuptools * error when trying to build wheels with old setuptools --- certbot/certbot/plugins/dns_test_common.py | 5 ++++- certbot/certbot/plugins/dns_test_common_lexicon.py | 5 ++++- certbot/certbot/tests/util.py | 5 ++++- certbot/setup.py | 12 ++++++++++-- certbot/tests/account_test.py | 5 ++++- certbot/tests/auth_handler_test.py | 5 ++++- certbot/tests/cert_manager_test.py | 5 ++++- certbot/tests/cli_test.py | 5 ++++- certbot/tests/client_test.py | 5 ++++- certbot/tests/compat/filesystem_test.py | 5 ++++- certbot/tests/configuration_test.py | 5 ++++- certbot/tests/crypto_util_test.py | 5 ++++- certbot/tests/display/completer_test.py | 5 ++++- certbot/tests/display/ops_test.py | 5 ++++- certbot/tests/display/util_test.py | 5 ++++- certbot/tests/eff_test.py | 5 ++++- certbot/tests/error_handler_test.py | 5 ++++- certbot/tests/errors_test.py | 5 ++++- certbot/tests/hook_test.py | 5 ++++- certbot/tests/lock_test.py | 5 ++++- certbot/tests/log_test.py | 5 ++++- certbot/tests/main_test.py | 5 ++++- certbot/tests/ocsp_test.py | 5 ++++- certbot/tests/plugins/common_test.py | 5 ++++- certbot/tests/plugins/disco_test.py | 5 ++++- certbot/tests/plugins/dns_common_lexicon_test.py | 5 ++++- certbot/tests/plugins/dns_common_test.py | 5 ++++- certbot/tests/plugins/enhancements_test.py | 5 ++++- certbot/tests/plugins/manual_test.py | 5 ++++- certbot/tests/plugins/null_test.py | 5 ++++- certbot/tests/plugins/selection_test.py | 5 ++++- certbot/tests/plugins/standalone_test.py | 5 ++++- certbot/tests/plugins/storage_test.py | 5 ++++- certbot/tests/plugins/util_test.py | 5 ++++- certbot/tests/plugins/webroot_test.py | 5 ++++- certbot/tests/renewal_test.py | 5 ++++- certbot/tests/renewupdater_test.py | 5 ++++- certbot/tests/reporter_test.py | 5 ++++- certbot/tests/reverter_test.py | 5 ++++- certbot/tests/storage_test.py | 5 ++++- certbot/tests/util_test.py | 5 ++++- 41 files changed, 170 insertions(+), 42 deletions(-) diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index 9ef76c2c3..d5044d336 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -2,7 +2,10 @@ import configobj import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six from acme import challenges diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index c77d6da9e..1bef06042 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -1,7 +1,10 @@ """Base test class for DNS authenticators built on Lexicon.""" import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 8b28b1080..92f52a852 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -10,7 +10,10 @@ import unittest from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import pkg_resources import six diff --git a/certbot/setup.py b/certbot/setup.py index 514aec8c5..143e1a10a 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -47,7 +47,6 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', @@ -62,7 +61,8 @@ install_requires = [ # So this dependency is not added for old Linux distributions with old setuptools, # in order to allow these systems to build certbot from sources. pywin32_req = 'pywin32>=227' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py -if StrictVersion(setuptools_version) >= StrictVersion('36.2'): +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: install_requires.append(pywin32_req + " ; sys_platform == 'win32'") elif 'bdist_wheel' in sys.argv[1:]: raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' @@ -73,6 +73,14 @@ elif os.name == 'nt': # setuptools, pywin32 will not be specified as a dependency. install_requires.append(pywin32_req) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'coverage', 'ipdb', diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 4a6ed3e01..6c6e6c860 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -4,7 +4,10 @@ import json import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz from acme import messages diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 7ab3a2baa..6cd207b4b 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -3,7 +3,10 @@ import functools import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from acme import challenges diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index bea64f09c..d956fd04f 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -7,7 +7,10 @@ import tempfile import unittest import configobj -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot._internal import configuration diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 7d21f8bb8..592c40be7 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -4,7 +4,10 @@ import copy import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 3e0e5b212..cbc058c7a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -5,7 +5,10 @@ import tempfile import unittest from josepy import interfaces -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index fdfb1ffe9..1c2d2df0d 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -3,7 +3,10 @@ import contextlib import errno import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import util from certbot._internal import lock diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index d748b9bfb..1f8a0e803 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.configuration.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot._internal import constants diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index d52e3acdb..481d83894 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -2,7 +2,10 @@ import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL import zope.component diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index a183fd14f..0852ab175 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -7,7 +7,10 @@ import string import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from six.moves import reload_module # pylint: disable=import-error from certbot.compat import filesystem # pylint: disable=ungrouped-imports diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 327f1bcbe..a683e1d3d 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,7 +4,10 @@ import sys import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from acme import messages diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index eccfee7db..3e492e9ab 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -4,7 +4,10 @@ import socket import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from certbot import errors diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index cdd7908a3..c4a25da69 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.eff.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import requests from certbot._internal import constants diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 899dbc611..e5a95c3a8 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -4,7 +4,10 @@ import signal import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.compat import os diff --git a/certbot/tests/errors_test.py b/certbot/tests/errors_test.py index d6c829322..792868df0 100644 --- a/certbot/tests/errors_test.py +++ b/certbot/tests/errors_test.py @@ -1,7 +1,10 @@ """Tests for certbot.errors.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import messages from certbot import achallenges diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 3b7a94489..32081f9d0 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.hooks.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 5a48009fd..2f887d33e 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -3,7 +3,10 @@ import functools import multiprocessing import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot.compat import os diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 5b0918ce5..5cd287c2e 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -5,7 +5,10 @@ import sys import time import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import messages diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 92afc3fef..8113b2bc4 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -13,7 +13,10 @@ import traceback import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 9eb49e115..af54844cf 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -10,7 +10,10 @@ from cryptography.exceptions import InvalidSignature from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes # type: ignore -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz from certbot import errors diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 1f339a6f3..344e6312f 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -4,7 +4,10 @@ import shutil import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges from certbot import achallenges diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index eec0795e3..5a0a392b0 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -3,7 +3,10 @@ import functools import string import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pkg_resources import six import zope.interface diff --git a/certbot/tests/plugins/dns_common_lexicon_test.py b/certbot/tests/plugins/dns_common_lexicon_test.py index 986362ca9..a67430f3e 100644 --- a/certbot/tests/plugins/dns_common_lexicon_test.py +++ b/certbot/tests/plugins/dns_common_lexicon_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.plugins import dns_common_lexicon from certbot.plugins import dns_test_common_lexicon diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index eba3c89d6..993f3b461 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -4,7 +4,10 @@ import collections import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/plugins/enhancements_test.py b/certbot/tests/plugins/enhancements_test.py index 05fbc5028..a20a6864f 100644 --- a/certbot/tests/plugins/enhancements_test.py +++ b/certbot/tests/plugins/enhancements_test.py @@ -1,7 +1,10 @@ """Tests for new style enhancements""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot._internal.plugins import null from certbot.plugins import enhancements diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 6cdef148a..933c759d6 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import challenges diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index db0213813..47708e340 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.plugins.null.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index c66473ad1..e5e6db031 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from certbot import errors diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 862accb92..751b9d943 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -5,7 +5,10 @@ from socket import errno as socket_errors # type: ignore import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL.crypto # pylint: disable=unused-import import six diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index e9ca2007f..4b0d1da83 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -2,7 +2,10 @@ import json import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot.compat import filesystem diff --git a/certbot/tests/plugins/util_test.py b/certbot/tests/plugins/util_test.py index c41e55222..9387b4ae7 100644 --- a/certbot/tests/plugins/util_test.py +++ b/certbot/tests/plugins/util_test.py @@ -1,7 +1,10 @@ """Tests for certbot.plugins.util.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.compat import os diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index fade12bb1..e57e09eae 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -10,7 +10,10 @@ import tempfile import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import challenges diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index e92211ea2..1fc54b42e 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.renewal""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges from certbot import errors diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index c6f8f3713..b5ecddb5a 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -1,7 +1,10 @@ """Tests for renewal updater interfaces""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import interfaces from certbot._internal import main diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 3d7c80172..7d03f1821 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index e592bcbdc..d67aa431a 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -5,7 +5,10 @@ import shutil import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from certbot import errors diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 0f7620b78..5aa37824d 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -6,7 +6,10 @@ import stat import unittest import configobj -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz import six diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 3ff09a83f..6dd0f964c 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -4,7 +4,10 @@ import errno import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from six.moves import reload_module # pylint: disable=import-error From 8fb9a395ab4393c5d06704de01687421f7acee1f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:30:08 -0700 Subject: [PATCH 67/72] Do not require mock in Python 3 in apache module (#7896) Part of #7886. This PR conditionally installs mock in `apache/setup.py` based on setuptools version and python version, when possible. It then updates `apache` tests to use `unittest.mock` when `mock` isn't available. * Conditionally install mock in apache * error out on newer python and older setuptools * error when trying to build wheels with old setuptools * use unittest.mock when third-party mock isn't available in apache, with no cover and type ignore --- certbot-apache/local-oldest-requirements.txt | 2 +- certbot-apache/setup.py | 12 +++++++++++- certbot-apache/tests/augeasnode_test.py | 5 ++++- certbot-apache/tests/autohsts_test.py | 5 ++++- certbot-apache/tests/centos_test.py | 5 ++++- certbot-apache/tests/configurator_reverter_test.py | 5 ++++- certbot-apache/tests/configurator_test.py | 5 ++++- certbot-apache/tests/debian_test.py | 5 ++++- certbot-apache/tests/display_ops_test.py | 5 ++++- certbot-apache/tests/dualnode_test.py | 5 ++++- certbot-apache/tests/entrypoint_test.py | 5 ++++- certbot-apache/tests/fedora_test.py | 5 ++++- certbot-apache/tests/gentoo_test.py | 5 ++++- certbot-apache/tests/http_01_test.py | 5 ++++- certbot-apache/tests/parser_test.py | 5 ++++- certbot-apache/tests/parsernode_configurator_test.py | 5 ++++- certbot-apache/tests/util.py | 5 ++++- 17 files changed, 72 insertions(+), 17 deletions(-) diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt index cf61c15a5..e45a0f831 100644 --- a/certbot-apache/local-oldest-requirements.txt +++ b/certbot-apache/local-oldest-requirements.txt @@ -1,3 +1,3 @@ # Remember to update setup.py to match the package versions below. acme[dev]==0.29.0 -certbot[dev]==1.1.0 +certbot[dev]==1.1.0 \ No newline at end of file diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 14300370a..25fb920c2 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,13 +13,21 @@ version = '1.4.0.dev0' install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', - 'mock', 'python-augeas', 'setuptools', 'zope.component', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'apacheconfig>=0.3.2', ] diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 270cc8c44..abe72a5d0 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -1,5 +1,8 @@ """Tests for AugeasParserNode classes""" -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import os import util diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index 8e4f15d6b..d15600215 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -3,7 +3,10 @@ import re import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six # pylint: disable=unused-import # six is used in mock.patch() from certbot import errors diff --git a/certbot-apache/tests/centos_test.py b/certbot-apache/tests/centos_test.py index 40d1361d4..9dc6fa5a7 100644 --- a/certbot-apache/tests/centos_test.py +++ b/certbot-apache/tests/centos_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.configurator for Centos overrides""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import filesystem diff --git a/certbot-apache/tests/configurator_reverter_test.py b/certbot-apache/tests/configurator_reverter_test.py index ad8e73347..8596195d8 100644 --- a/certbot-apache/tests/configurator_reverter_test.py +++ b/certbot-apache/tests/configurator_reverter_test.py @@ -2,7 +2,10 @@ import shutil import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors import util diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 694c1c0bb..8fd3cb750 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -6,7 +6,10 @@ import socket import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six # pylint: disable=unused-import # six is used in mock.patch() from acme import challenges diff --git a/certbot-apache/tests/debian_test.py b/certbot-apache/tests/debian_test.py index 7a65bb7b1..192e3cba5 100644 --- a/certbot-apache/tests/debian_test.py +++ b/certbot-apache/tests/debian_test.py @@ -2,7 +2,10 @@ import shutil import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-apache/tests/display_ops_test.py b/certbot-apache/tests/display_ops_test.py index e9f54a562..4559668ac 100644 --- a/certbot-apache/tests/display_ops_test.py +++ b/certbot-apache/tests/display_ops_test.py @@ -1,7 +1,10 @@ """Test certbot_apache._internal.display_ops.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.display import util as display_util diff --git a/certbot-apache/tests/dualnode_test.py b/certbot-apache/tests/dualnode_test.py index 0871bac78..44cc69ff4 100644 --- a/certbot-apache/tests/dualnode_test.py +++ b/certbot-apache/tests/dualnode_test.py @@ -1,7 +1,10 @@ """Tests for DualParserNode implementation""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot_apache._internal import assertions from certbot_apache._internal import augeasparser diff --git a/certbot-apache/tests/entrypoint_test.py b/certbot-apache/tests/entrypoint_test.py index 04c393bdf..6f6f5bbb0 100644 --- a/certbot-apache/tests/entrypoint_test.py +++ b/certbot-apache/tests/entrypoint_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.entrypoint for override class resolution""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot_apache._internal import configurator from certbot_apache._internal import entrypoint diff --git a/certbot-apache/tests/fedora_test.py b/certbot-apache/tests/fedora_test.py index 7f1d6526f..e0ee603c3 100644 --- a/certbot-apache/tests/fedora_test.py +++ b/certbot-apache/tests/fedora_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.configurator for Fedora 29+ overrides""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import filesystem diff --git a/certbot-apache/tests/gentoo_test.py b/certbot-apache/tests/gentoo_test.py index 5a35c17ae..aa923c367 100644 --- a/certbot-apache/tests/gentoo_test.py +++ b/certbot-apache/tests/gentoo_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.configurator for Gentoo overrides""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import filesystem diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 7d9019702..696cd4a54 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -2,7 +2,10 @@ import unittest import errno -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from acme import challenges from certbot import achallenges diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py index 299eb4567..7aedec31d 100644 --- a/certbot-apache/tests/parser_test.py +++ b/certbot-apache/tests/parser_test.py @@ -2,7 +2,10 @@ import shutil import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-apache/tests/parsernode_configurator_test.py b/certbot-apache/tests/parsernode_configurator_test.py index afaaa6e43..7fbec2540 100644 --- a/certbot-apache/tests/parsernode_configurator_test.py +++ b/certbot-apache/tests/parsernode_configurator_test.py @@ -1,7 +1,10 @@ """Tests for ApacheConfigurator for AugeasParserNode classes""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import util diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index a39e5225d..f2a6a0263 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -5,7 +5,10 @@ import unittest import augeas import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import zope.component from certbot.compat import os From 49912732aca0903915c95011d9602423e1174c8f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:39:44 -0700 Subject: [PATCH 68/72] Do not require mock in Python 3 in nginx module (#7898) * Do not require mock in Python 3 in nginx module * error when trying to build wheels with old setuptools * add type: ignores --- certbot-nginx/setup.py | 12 +++++++++++- certbot-nginx/tests/configurator_test.py | 5 ++++- certbot-nginx/tests/http_01_test.py | 5 ++++- certbot-nginx/tests/parser_obj_test.py | 5 ++++- certbot-nginx/tests/test_util.py | 5 ++++- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 0d62e7d55..0e6deceb3 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,13 +13,21 @@ version = '1.4.0.dev0' install_requires = [ 'acme>=1.4.0.dev0', 'certbot>=1.4.0.dev0', - 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + class PyTest(TestCommand): user_options = [] diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 0a04a22a4..2c3264a5f 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -1,7 +1,10 @@ """Test for certbot_nginx._internal.configurator.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL from acme import challenges diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index 6418a8841..8f0673c1f 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -2,7 +2,10 @@ import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six from acme import challenges diff --git a/certbot-nginx/tests/parser_obj_test.py b/certbot-nginx/tests/parser_obj_test.py index bb7834701..8262c5f52 100644 --- a/certbot-nginx/tests/parser_obj_test.py +++ b/certbot-nginx/tests/parser_obj_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot_nginx._internal.parser_obj import COMMENT_BLOCK from certbot_nginx._internal.parser_obj import parse_raw diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index 4c9da84bd..4b26f7935 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -4,7 +4,10 @@ import shutil import tempfile import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import pkg_resources import zope.component From af21d1d56ece276e4a259ceba894e4d008446827 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:40:03 -0700 Subject: [PATCH 69/72] Do not require mock in Python 3 in certbot-compatibility-test module (#7899) * Do not require mock in Python 3 in certbot-compatibility-test module * error when trying to build wheels with old setuptools * add type: ignores --- .../configurators/apache/common.py | 5 ++++- .../certbot_compatibility_test/validator_test.py | 5 ++++- certbot-compatibility-test/setup.py | 12 +++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index a9b1ce87e..5d5542ffd 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -3,7 +3,10 @@ import os import shutil import subprocess -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import zope.interface from certbot import errors as le_errors diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator_test.py b/certbot-compatibility-test/certbot_compatibility_test/validator_test.py index 235ce0e3c..0b1056561 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator_test.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator_test.py @@ -1,7 +1,10 @@ """Tests for certbot_compatibility_test.validator.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import requests diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index d6760576a..ed8e3f861 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup @@ -8,12 +10,20 @@ version = '1.4.0.dev0' install_requires = [ 'certbot', 'certbot-apache', - 'mock', 'six', 'requests', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + if sys.version_info < (2, 7, 9): # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) install_requires.append('ndg-httpsclient') From 9c345ac301287fe8f3222a1084aa79a0dbca13e3 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:54:44 -0700 Subject: [PATCH 70/72] Do not require mock in Python 3 in certbot-dns modules (#7900) Part of #7886. This PR conditionally installs `mock` in `certbot-dns-*/setup.py` based on setuptools version and python version, when possible. It then updates the tests to use `unittest.mock` when `mock` isn't available. * Do not require mock in Python 3 in certbot-dns modules * update changelog * error when trying to build wheels with old setuptools * add type: ignores --- certbot-dns-cloudflare/setup.py | 12 +++++++++++- certbot-dns-cloudflare/tests/dns_cloudflare_test.py | 5 ++++- certbot-dns-cloudxns/setup.py | 12 +++++++++++- certbot-dns-cloudxns/tests/dns_cloudxns_test.py | 5 ++++- certbot-dns-digitalocean/setup.py | 12 +++++++++++- .../tests/dns_digitalocean_test.py | 5 ++++- certbot-dns-dnsimple/setup.py | 12 +++++++++++- certbot-dns-dnsimple/tests/dns_dnsimple_test.py | 5 ++++- certbot-dns-dnsmadeeasy/setup.py | 12 +++++++++++- .../tests/dns_dnsmadeeasy_test.py | 5 ++++- certbot-dns-gehirn/setup.py | 12 +++++++++++- certbot-dns-gehirn/tests/dns_gehirn_test.py | 5 ++++- certbot-dns-google/setup.py | 12 +++++++++++- certbot-dns-google/tests/dns_google_test.py | 5 ++++- certbot-dns-linode/setup.py | 12 +++++++++++- certbot-dns-linode/tests/dns_linode_test.py | 5 ++++- certbot-dns-luadns/setup.py | 12 +++++++++++- certbot-dns-luadns/tests/dns_luadns_test.py | 5 ++++- certbot-dns-nsone/setup.py | 12 +++++++++++- certbot-dns-nsone/tests/dns_nsone_test.py | 5 ++++- certbot-dns-ovh/setup.py | 12 +++++++++++- certbot-dns-ovh/tests/dns_ovh_test.py | 5 ++++- certbot-dns-rfc2136/setup.py | 12 +++++++++++- certbot-dns-rfc2136/tests/dns_rfc2136_test.py | 5 ++++- certbot-dns-route53/setup.py | 12 +++++++++++- certbot-dns-route53/tests/dns_route53_test.py | 5 ++++- certbot-dns-sakuracloud/setup.py | 12 +++++++++++- .../tests/dns_sakuracloud_test.py | 5 ++++- certbot/CHANGELOG.md | 1 + 29 files changed, 211 insertions(+), 28 deletions(-) diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 67aac3231..9ac16bc67 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'cloudflare>=1.5.1', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py index d38330191..4d2dcf4ca 100644 --- a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py +++ b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py @@ -3,7 +3,10 @@ import unittest import CloudFlare -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 6c653967d..f998027f9 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py index a1e3cde89..43c69790f 100644 --- a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py +++ b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index c45fc8d03..7aef67d75 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,13 +13,21 @@ version = '1.4.0.dev0' install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', - 'mock', 'python-digitalocean>=1.11', 'setuptools', 'six', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py index 71301a47c..a752f52d0 100644 --- a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py @@ -3,7 +3,10 @@ import unittest import digitalocean -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 9124f0552..4a3f863f5 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -1,6 +1,8 @@ +from distutils.version import StrictVersion import os import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ version = '1.4.0.dev0' install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + # This package normally depends on dns-lexicon>=3.2.1 to address the # problem described in https://github.com/AnalogJ/lexicon/issues/387, # however, the fix there has been backported to older versions of diff --git a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py index ca5eb4f36..40eba4754 100644 --- a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py +++ b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2a4fd92b0..2dced23bf 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py index b94cc7d05..4a69e977c 100644 --- a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py +++ b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 2198fdd3e..7c7acc503 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,11 +13,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.1.22', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-gehirn/tests/dns_gehirn_test.py b/certbot-dns-gehirn/tests/dns_gehirn_test.py index f5b95b6c3..0598a5eb5 100644 --- a/certbot-dns-gehirn/tests/dns_gehirn_test.py +++ b/certbot-dns-gehirn/tests/dns_gehirn_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 087766edd..1b51a781e 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,7 +14,6 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'google-api-python-client>=1.5.5', - 'mock', 'oauth2client>=4.0', 'setuptools', 'zope.interface', @@ -20,6 +21,15 @@ install_requires = [ 'httplib2' ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 647a75b05..5af027cef 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -6,7 +6,10 @@ from googleapiclient import discovery from googleapiclient.errors import Error from googleapiclient.http import HttpMock from httplib2 import ServerNotFoundError -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 1e6b96b71..860c40079 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,11 +13,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.3', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-linode/tests/dns_linode_test.py b/certbot-dns-linode/tests/dns_linode_test.py index 3cf615486..fb9b1aa93 100644 --- a/certbot-dns-linode/tests/dns_linode_test.py +++ b/certbot-dns-linode/tests/dns_linode_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4db50b56c..83932e140 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-luadns/tests/dns_luadns_test.py b/certbot-dns-luadns/tests/dns_luadns_test.py index 934d3e103..a1242582f 100644 --- a/certbot-dns-luadns/tests/dns_luadns_test.py +++ b/certbot-dns-luadns/tests/dns_luadns_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 49e5e3bcf..459c6d752 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-nsone/tests/dns_nsone_test.py b/certbot-dns-nsone/tests/dns_nsone_test.py index dd6168f08..83371252f 100644 --- a/certbot-dns-nsone/tests/dns_nsone_test.py +++ b/certbot-dns-nsone/tests/dns_nsone_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 6c66b39dc..5823b237e 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-ovh/tests/dns_ovh_test.py b/certbot-dns-ovh/tests/dns_ovh_test.py index a420239ab..dd0f3058b 100644 --- a/certbot-dns-ovh/tests/dns_ovh_test.py +++ b/certbot-dns-ovh/tests/dns_ovh_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 5e0900e4d..94cda6f65 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'dnspython', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py index c767dba23..4c14a8072 100644 --- a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py @@ -5,7 +5,10 @@ import unittest import dns.flags import dns.rcode import dns.tsig -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 98455c362..f140d3f8d 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'boto3', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + class PyTest(TestCommand): user_options = [] diff --git a/certbot-dns-route53/tests/dns_route53_test.py b/certbot-dns-route53/tests/dns_route53_test.py index 85ec259b1..a77495313 100644 --- a/certbot-dns-route53/tests/dns_route53_test.py +++ b/certbot-dns-route53/tests/dns_route53_test.py @@ -4,7 +4,10 @@ import unittest from botocore.exceptions import ClientError from botocore.exceptions import NoCredentialsError -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 16990a4d3..b57e28577 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,11 +13,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.1.23', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py index 16890b5a9..af94336b3 100644 --- a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py +++ b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index f61cdcfc7..1b9379de7 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -19,6 +19,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * Stop asking interactively if the user would like to add a redirect. +* `mock` dependency is now conditional on Python 2 in all of our packages. ### Fixed From f66314926aa842804b8b64af785f2cef5296e0c5 Mon Sep 17 00:00:00 2001 From: April King Date: Wed, 15 Apr 2020 15:54:17 -0500 Subject: [PATCH 71/72] Update URL for Mozilla SSL Configuration Generator (#7912) --- AUTHORS.md | 1 + certbot/docs/ciphers.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS.md b/AUTHORS.md index 4414076fc..21a6e7773 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -21,6 +21,7 @@ Authors * [Andrzej G처rski](https://github.com/andrzej3393) * [Anselm Levskaya](https://github.com/levskaya) * [Antoine Jacoutot](https://github.com/ajacoutot) +* [April King](https://github.com/april) * [asaph](https://github.com/asaph) * [Axel Beckert](https://github.com/xtaran) * [Bas](https://github.com/Mechazawa) diff --git a/certbot/docs/ciphers.rst b/certbot/docs/ciphers.rst index 04b24b526..325d6244c 100644 --- a/certbot/docs/ciphers.rst +++ b/certbot/docs/ciphers.rst @@ -241,7 +241,7 @@ Mozilla Mozilla's general server configuration guidance is available at https://wiki.mozilla.org/Security/Server_Side_TLS -Mozilla has also produced a configuration generator: https://mozilla.github.io/server-side-tls/ssl-config-generator/ +Mozilla has also produced a configuration generator: https://ssl-config.mozilla.org Dutch National Cyber Security Centre ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 859dc38cb9195de072bc46e30e3edc0dab04f84d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 16 Apr 2020 08:59:40 -0700 Subject: [PATCH 72/72] Consolidate cover envs and default to py3-cover (#7905) * Consolidate cover envs and default to py3-cover * use py38 for code coverage in Travis * Disable coverage on Python < 3.6 line. --- .travis.yml | 10 ++++---- .../certbot_apache/_internal/parser.py | 2 +- tox.ini | 23 +++++-------------- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae25a6895..d3eeb1e03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,8 +44,8 @@ matrix: <<: *not-on-master # This job is always executed, including on master - - python: "2.7" - env: TOXENV=py27-cover FYI="py27 tests + code coverage" + - python: "3.8" + env: TOXENV=py38-cover FYI="py38 tests + code coverage" - python: "3.7" env: TOXENV=lint @@ -60,12 +60,12 @@ matrix: dist: trusty env: TOXENV='py27-{acme,apache,apache-v2,certbot,dns,nginx}-oldest' <<: *not-on-master + - python: "2.7" + env: TOXENV=py27 + <<: *not-on-master - python: "3.5" env: TOXENV=py35 <<: *not-on-master - - python: "3.8" - env: TOXENV=py38 - <<: *not-on-master - sudo: required env: TOXENV=apache_compat services: docker diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index f6aa3fe48..c9aebae54 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -741,7 +741,7 @@ class ApacheParser(object): """ if sys.version_info < (3, 6): # This strips off final /Z(?ms) - return fnmatch.translate(clean_fn_match)[:-7] + return fnmatch.translate(clean_fn_match)[:-7] # pragma: no cover # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z return fnmatch.translate(clean_fn_match)[4:-3] # pragma: no cover diff --git a/tox.ini b/tox.ini index 8aa4bfbf2..7f5b7bd5a 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py3,py27-cover,lint,mypy +envlist = modification,py3-cover,lint,mypy [base] # pip installs the requested packages in editable mode @@ -63,8 +63,11 @@ source_paths = passenv = CERTBOT_NO_PIN commands = - {[base]install_and_test} {[base]all_packages} - python tests/lock_test.py + !cover: {[base]install_and_test} {[base]all_packages} + !cover: python tests/lock_test.py + cover: {[base]install_packages} + cover: {[base]pip_install} certbot-apache[dev] + cover: python tox.cover.py # We always recreate the virtual environment to avoid problems like # https://github.com/certbot/certbot/issues/7745. recreate = true @@ -116,20 +119,6 @@ commands = setenv = {[testenv:py27-oldest]setenv} -[testenv:py27-cover] -basepython = python2.7 -commands = - {[base]install_packages} - {[base]pip_install} certbot-apache[dev] - python tox.cover.py - -[testenv:py37-cover] -basepython = python3.7 -commands = - {[base]install_packages} - {[base]pip_install} certbot-apache[dev] - python tox.cover.py - [testenv:lint] basepython = python3 # separating into multiple invocations disables cross package