From 103039ca40c3a300816c5a957a0bf13a033234de Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 Jan 2018 17:46:56 -0800 Subject: [PATCH 01/93] Add 0.21.0 changelog --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4acfc0401..f6f7be9fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,51 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.21.0 - 2018-01-17 + +### Added + +* Support for the HTTP-01 challenge type was added to our Apache and Nginx + plugins. For those not aware, Let's Encrypt disabled the TLS-SNI-01 challenge + type which was what was previously being used by our Apache and Nginx plugins + last week due to a security issue. For more information about Let's Encrypt's + change, click + [here](https://community.letsencrypt.org/t/2018-01-11-update-regarding-acme-tls-sni-and-shared-hosting-infrastructure/50188). + Our Apache and Nginx plugins will automatically switch to use HTTP-01 so no + changes need to be made to your Certbot configuration, however, you should + make sure your server is accessible on port 80 and isn't behind an external + proxy doing things like redirecting all traffic from HTTP to HTTPS. HTTP to + HTTPS redirects inside Apache and Nginx are fine. +* IPv6 support was added to the Nginx plugin. +* Support for automatically creating server blocks based on the default server + block was added to the Nginx plugin. +* The flags --delete-after-revoke and --no-delete-after-revoke were added + allowing users to control whether the revoke subcommand also deletes the + certificates it is revoking. + +### Changed + +* We deprecated support for Python 2.6 and Python 3.3 in Certbot and its ACME + library. Support for these versions of Python will be removed in the next + major release of Certbot. If you are using certbot-auto on a RHEL 6 based + system, it will guide you through the process of installing Python 3. +* We split our implementation of JOSE (Javascript Object Signing and + Encryption) out of our ACME library and into a separate package named josepy. + This package is available on [PyPI](https://pypi.python.org/pypi/josepy) and + on [GitHub](https://github.com/certbot/josepy). +* We updated the ciphersuites used in Apache to the new [values recommended by + Mozilla](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29). + The major change here is adding ChaCha20 to the list of supported + ciphersuites. + +### Fixed + +* An issue with our Apache plugin on Gentoo due to differences in their + apache2ctl command have been resolved. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/47?closed=1 + ## 0.20.0 - 2017-12-06 ### Added From b0aa8b7c0bcddb0837d9a57d021152bc05473b8e Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 24 Jan 2018 02:46:36 +0200 Subject: [PATCH 02/93] Work around Basic Authentication for challenge dir in Apache (#5461) Unfortunately, the way that Apache merges the configuration directives is different for mod_rewrite and / directives. To work around basic auth in VirtualHosts, the challenge override Include had to be split in two. The first part handles overrides for RewriteRule and the other part will handle overrides for and directives. --- certbot-apache/certbot_apache/http_01.py | 60 +++++++++++++------ .../certbot_apache/tests/http_01_test.py | 24 +++++--- 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index e463f3880..cce93a646 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -11,30 +11,43 @@ logger = logging.getLogger(__name__) class ApacheHttp01(common.TLSSNI01): """Class that performs HTTP-01 challenges within the Apache configurator.""" - CONFIG_TEMPLATE22 = """\ + CONFIG_TEMPLATE22_PRE = """\ RewriteEngine on RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L] + """ + CONFIG_TEMPLATE22_POST = """\ Order Allow,Deny Allow from all + + Order Allow,Deny + Allow from all + """ - CONFIG_TEMPLATE24 = """\ + CONFIG_TEMPLATE24_PRE = """\ RewriteEngine on RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END] - + """ + CONFIG_TEMPLATE24_POST = """\ Require all granted + + Require all granted + """ def __init__(self, *args, **kwargs): super(ApacheHttp01, self).__init__(*args, **kwargs) - self.challenge_conf = os.path.join( + self.challenge_conf_pre = os.path.join( self.configurator.conf("challenge-location"), - "le_http_01_challenge.conf") + "le_http_01_challenge_pre.conf") + self.challenge_conf_post = os.path.join( + self.configurator.conf("challenge-location"), + "le_http_01_challenge_post.conf") self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") @@ -79,24 +92,32 @@ class ApacheHttp01(common.TLSSNI01): chall.domain, filter_defaults=False, port=str(self.configurator.config.http01_port)) if vh: - self._set_up_include_directive(vh) + self._set_up_include_directives(vh) else: for vh in self._relevant_vhosts(): - self._set_up_include_directive(vh) + self._set_up_include_directives(vh) self.configurator.reverter.register_file_creation( - True, self.challenge_conf) + True, self.challenge_conf_pre) + self.configurator.reverter.register_file_creation( + True, self.challenge_conf_post) if self.configurator.version < (2, 4): - config_template = self.CONFIG_TEMPLATE22 + config_template_pre = self.CONFIG_TEMPLATE22_PRE + config_template_post = self.CONFIG_TEMPLATE22_POST else: - config_template = self.CONFIG_TEMPLATE24 + config_template_pre = self.CONFIG_TEMPLATE24_PRE + config_template_post = self.CONFIG_TEMPLATE24_POST - config_text = config_template.format(self.challenge_dir) + config_text_pre = config_template_pre.format(self.challenge_dir) + config_text_post = config_template_post.format(self.challenge_dir) - logger.debug("writing a config file with text:\n %s", config_text) - with open(self.challenge_conf, "w") as new_conf: - new_conf.write(config_text) + logger.debug("writing a pre config file with text:\n %s", config_text_pre) + with open(self.challenge_conf_pre, "w") as new_conf: + new_conf.write(config_text_pre) + logger.debug("writing a post config file with text:\n %s", config_text_post) + with open(self.challenge_conf_post, "w") as new_conf: + new_conf.write(config_text_post) def _relevant_vhosts(self): http01_port = str(self.configurator.config.http01_port) @@ -137,14 +158,17 @@ class ApacheHttp01(common.TLSSNI01): return response - def _set_up_include_directive(self, vhost): - """Includes override configuration to the beginning of VirtualHost. - Note that this include isn't added to Augeas search tree""" + def _set_up_include_directives(self, vhost): + """Includes override configuration to the beginning and to the end of + VirtualHost. Note that this include isn't added to Augeas search tree""" if vhost not in self.moded_vhosts: logger.debug( "Adding a temporary challenge validation Include for name: %s " + "in: %s", vhost.name, vhost.filep) self.configurator.parser.add_dir_beginning( - vhost.path, "Include", self.challenge_conf) + vhost.path, "Include", self.challenge_conf_pre) + self.configurator.parser.add_dir( + vhost.path, "Include", self.challenge_conf_post) + self.moded_vhosts.add(vhost) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 64a76649a..9ed4ee509 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -158,23 +158,31 @@ class ApacheHttp01Test(util.ApacheTest): for vhost in vhosts: if not vhost.ssl: matches = self.config.parser.find_dir("Include", - self.http.challenge_conf, + self.http.challenge_conf_pre, + vhost.path) + self.assertEqual(len(matches), 1) + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf_post, vhost.path) self.assertEqual(len(matches), 1) self.assertTrue(os.path.exists(challenge_dir)) def _test_challenge_conf(self): - with open(self.http.challenge_conf) as f: - conf_contents = f.read() + with open(self.http.challenge_conf_pre) as f: + pre_conf_contents = f.read() - self.assertTrue("RewriteEngine on" in conf_contents) - self.assertTrue("RewriteRule" in conf_contents) - self.assertTrue(self.http.challenge_dir in conf_contents) + with open(self.http.challenge_conf_post) as f: + post_conf_contents = f.read() + + self.assertTrue("RewriteEngine on" in pre_conf_contents) + self.assertTrue("RewriteRule" in pre_conf_contents) + + self.assertTrue(self.http.challenge_dir in post_conf_contents) if self.config.version < (2, 4): - self.assertTrue("Allow from all" in conf_contents) + self.assertTrue("Allow from all" in post_conf_contents) else: - self.assertTrue("Require all granted" in conf_contents) + self.assertTrue("Require all granted" in post_conf_contents) def _test_challenge_file(self, achall): name = os.path.join(self.http.challenge_dir, achall.chall.encode("token")) From c0068791ce2a4edbe4b7293c6eab21d8b62347ca Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 24 Jan 2018 13:56:40 -0800 Subject: [PATCH 03/93] add let's encrypt status to footer and fix link --- docs/_templates/footer.html | 52 +++++++++++++++++++++++++++++++++++++ docs/conf.py | 3 ++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 docs/_templates/footer.html diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html new file mode 100644 index 000000000..8fd0f127d --- /dev/null +++ b/docs/_templates/footer.html @@ -0,0 +1,52 @@ +
+ {% if (theme_prev_next_buttons_location == 'bottom' or theme_prev_next_buttons_location == 'both') and (next or prev) %} + + {% endif %} + +
+ +
+

+ + © Copyright 2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license. + +
+
+ + Let's Encrypt Status + + + {%- if build_id and build_url %} + {% trans build_url=build_url, build_id=build_id %} + + Build + {{ build_id }}. + + {% endtrans %} + {%- elif commit %} + {% trans commit=commit %} + + Revision {{ commit }}. + + {% endtrans %} + {%- elif last_updated %} + {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} + {%- endif %} + +

+
+ + {%- if show_sphinx %} + {% trans %}Built with Sphinx using a theme provided by Read the Docs{% endtrans %}. + {%- endif %} + + {%- block extrafooter %} {% endblock %} + +
diff --git a/docs/conf.py b/docs/conf.py index 73df47dbd..09bb44285 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,7 +64,8 @@ master_doc = 'index' # General information about the project. project = u'Certbot' -copyright = u'2014-2016 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license ' +# this is now overridden by the footer.html template +#copyright = u'2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 0a4f926b160cf118c81974e91df7f196bc7a379d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 24 Jan 2018 15:01:42 -0800 Subject: [PATCH 04/93] Remove Default Detector log line. (#5372) This produces a super-long log line that wraps to 30-60 lines, depending on screen width. Even though it's just at debug level, it clutters up the integration test output without providing proportional debugging value. * Remove Default Detector log line. This produces about 30 lines of log output. Even though it's just at debug level, it clutters up the integration test output without providing proportional debugging value. * Add more useful logs. --- certbot/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index f0fa7eb7e..dec7474f9 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -207,13 +207,15 @@ def set_by_cli(var): # propagate plugin requests: eg --standalone modifies config.authenticator detector.authenticator, detector.installer = ( plugin_selection.cli_plugin_requests(detector)) - logger.debug("Default Detector is %r", detector) if not isinstance(getattr(detector, var), _Default): + logger.debug("Var %s=%s (set by user).", var, getattr(detector, var)) return True for modifier in VAR_MODIFIERS.get(var, []): if set_by_cli(modifier): + logger.debug("Var %s=%s (set by user).", + var, VAR_MODIFIERS.get(var, [])) return True return False From 8a9f21cdd36fb9a424430f9c807bacb27c1cfbf4 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 24 Jan 2018 22:19:32 -0800 Subject: [PATCH 05/93] Fix Nginx redirect issue (#5479) * wrap redirect in if host matches * return 404 if we've created a new block * change domain matching to exact match * insert new redirect directive at the top * add a redirect block to the top if it doesn't already exist, even if there's an existing redirect * fix obj tests * remove active parameter * update tests * add back spaces * move imports * remove unused code --- certbot-nginx/certbot_nginx/configurator.py | 50 ++++---- certbot-nginx/certbot_nginx/obj.py | 21 ---- .../certbot_nginx/tests/configurator_test.py | 110 +++--------------- certbot-nginx/certbot_nginx/tests/obj_test.py | 14 +-- 4 files changed, 44 insertions(+), 151 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index bb2933a39..9f091c0fd 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -31,16 +31,6 @@ from certbot_nginx import http_01 logger = logging.getLogger(__name__) -REDIRECT_BLOCK = [ - ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], - ['\n'] -] - -REDIRECT_COMMENT_BLOCK = [ - ['\n ', '#', ' Redirect non-https traffic to https'], - ['\n ', '#', ' return 301 https://$host$request_uri;'], - ['\n'] -] @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) @@ -571,24 +561,17 @@ class NginxConfigurator(common.Installer): logger.warning("Failed %s for %s", enhancement, domain) raise - def _has_certbot_redirect(self, vhost): - test_redirect_block = _test_block_from_block(REDIRECT_BLOCK) + def _has_certbot_redirect(self, vhost, domain): + test_redirect_block = _test_block_from_block(_redirect_block_for_domain(domain)) return vhost.contains_list(test_redirect_block) - def _has_certbot_redirect_comment(self, vhost): - test_redirect_comment_block = _test_block_from_block(REDIRECT_COMMENT_BLOCK) - return vhost.contains_list(test_redirect_comment_block) - - def _add_redirect_block(self, vhost, active=True): + def _add_redirect_block(self, vhost, domain): """Add redirect directive to vhost """ - if active: - redirect_block = REDIRECT_BLOCK - else: - redirect_block = REDIRECT_COMMENT_BLOCK + redirect_block = _redirect_block_for_domain(domain) self.parser.add_server_directives( - vhost, redirect_block, replace=False) + vhost, redirect_block, replace=False, insert_at_top=True) def _enable_redirect(self, domain, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. @@ -615,6 +598,7 @@ class NginxConfigurator(common.Installer): self.DEFAULT_LISTEN_PORT) return + new_vhost = None if vhost.ssl: new_vhost = self.parser.duplicate_vhost(vhost, only_directives=['listen', 'server_name']) @@ -631,20 +615,18 @@ class NginxConfigurator(common.Installer): # remove all non-ssl addresses from the existing block self.parser.remove_server_directives(vhost, 'listen', match_func=_no_ssl_match_func) + # Add this at the bottom to get the right order of directives + return_404_directive = [['\n ', 'return', ' ', '404']] + self.parser.add_server_directives(new_vhost, return_404_directive, replace=False) + vhost = new_vhost - if self._has_certbot_redirect(vhost): + if self._has_certbot_redirect(vhost, domain): logger.info("Traffic on port %s already redirecting to ssl in %s", self.DEFAULT_LISTEN_PORT, vhost.filep) - elif vhost.has_redirect(): - if not self._has_certbot_redirect_comment(vhost): - self._add_redirect_block(vhost, active=False) - logger.info("The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.", vhost.filep) else: # Redirect plaintextish host to https - self._add_redirect_block(vhost, active=True) + self._add_redirect_block(vhost, domain) logger.info("Redirecting all traffic on port %s to ssl in %s", self.DEFAULT_LISTEN_PORT, vhost.filep) @@ -907,6 +889,14 @@ def _test_block_from_block(block): parser.comment_directive(test_block, 0) return test_block[:-1] +def _redirect_block_for_domain(domain): + redirect_block = [[ + ['\n ', 'if', ' ', '($host', ' ', '=', ' ', '%s)' % domain, ' '], + [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + '\n ']], + ['\n']] + return redirect_block + def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index f5ac5c2e3..e8dc8936d 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -193,15 +193,6 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return False - def has_redirect(self): - """Determine if this vhost has a redirecting statement - """ - for directive_name in REDIRECT_DIRECTIVES: - found = _find_directive(self.raw, directive_name) - if found is not None: - return True - return False - def contains_list(self, test): """Determine if raw server block contains test list at top level """ @@ -225,15 +216,3 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods for a in self.addrs: if not a.ipv6: return True - -def _find_directive(directives, directive_name): - """Find a directive of type directive_name in directives - """ - if not directives or isinstance(directives, six.string_types) or len(directives) == 0: - return None - - if directives[0] == directive_name: - return directives - - matches = (_find_directive(line, directive_name) for line in directives) - return next((m for m in matches if m is not None), None) diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 7475df40c..acb7ee282 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -18,6 +18,8 @@ from certbot.tests import util as certbot_test_util from certbot_nginx import constants from certbot_nginx import obj from certbot_nginx import parser +from certbot_nginx.configurator import _redirect_block_for_domain +from certbot_nginx.nginxparser import UnspacedList from certbot_nginx.tests import util @@ -447,7 +449,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_redirect_enhance(self): # Test that we successfully add a redirect when there is # a listen directive - expected = ['return', '301', 'https://$host$request_uri'] + expected = UnspacedList(_redirect_block_for_domain("www.example.com"))[0] example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("www.example.com", "redirect") @@ -460,6 +462,8 @@ class NginxConfiguratorTest(util.NginxTest): migration_conf = self.config.parser.abs_path('sites-enabled/migration.com') self.config.enhance("migration.com", "redirect") + expected = UnspacedList(_redirect_block_for_domain("migration.com"))[0] + generated_conf = self.config.parser.parsed[migration_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) @@ -484,101 +488,27 @@ class NginxConfiguratorTest(util.NginxTest): ['ssl_dhparam', self.config.ssl_dhparams], ['#', ' managed by Certbot'], [], []]], [['server'], [ + [['if', '($host', '=', 'www.example.com)'], [ + ['return', '301', 'https://$host$request_uri']]], + ['#', ' managed by Certbot'], [], ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], ['server_name', 'example.*'], - ['return', '301', 'https://$host$request_uri'], ['#', ' managed by Certbot'], - [], []]]], + ['return', '404'], ['#', ' managed by Certbot'], [], [], []]]], generated_conf) @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): + def test_certbot_redirect_exists(self, mock_contains_list): # Test that we add no redirect statement if there is already a # redirect in the block that is managed by certbot # Has a certbot redirect - mock_has_redirect.return_value = True mock_contains_list.return_value = True with mock.patch("certbot_nginx.configurator.logger") as mock_logger: self.config.enhance("www.example.com", "redirect") self.assertEqual(mock_logger.info.call_args[0][0], "Traffic on port %s already redirecting to ssl in %s") - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_non_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): - # Test that we add a redirect as a comment if there is already a - # redirect-class statement in the block that isn't managed by certbot - example_conf = self.config.parser.abs_path('sites-enabled/example.com') - - # Has a non-Certbot redirect, and has no existing comment - mock_contains_list.return_value = False - mock_has_redirect.return_value = True - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertEqual(mock_logger.info.call_args[0][0], - "The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.") - generated_conf = self.config.parser.parsed[example_conf] - expected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' return 301 https://$host$request_uri;'], - ] - for line in expected: - self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) - - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_non_certbot_redirect_exists_has_ssl_copy(self, mock_has_redirect, mock_contains_list): - # Test that we add a redirect as a comment if there is already a - # redirect-class statement in the block that isn't managed by certbot - example_conf = self.config.parser.abs_path('sites-enabled/example.com') - - self.config.deploy_cert( - "example.org", - "example/cert.pem", - "example/key.pem", - "example/chain.pem", - "example/fullchain.pem") - - # Has a non-Certbot redirect, and has no existing comment - mock_contains_list.return_value = False - mock_has_redirect.return_value = True - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertEqual(mock_logger.info.call_args[0][0], - "The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.") - generated_conf = self.config.parser.parsed[example_conf] - expected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' return 301 https://$host$request_uri;'], - ] - for line in expected: - self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) - - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - @mock.patch('certbot_nginx.configurator.NginxConfigurator._has_certbot_redirect_comment') - @mock.patch('certbot_nginx.configurator.NginxConfigurator._add_redirect_block') - def test_redirect_comment_exists(self, mock_add_redirect_block, - mock_has_cb_redirect_comment, mock_has_redirect, mock_contains_list): - # Test that we add nothing if there is a non-Certbot redirect and a - # preexisting comment - # Has a non-Certbot redirect and a comment - mock_has_redirect.return_value = True - mock_contains_list.return_value = False # self._has_certbot_redirect(vhost): - mock_has_cb_redirect_comment.return_value = True - - # assert _add_redirect_block not called - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertFalse(mock_add_redirect_block.called) - self.assertTrue(mock_logger.info.called) - def test_redirect_dont_enhance(self): # Test that we don't accidentally add redirect to ssl-only block with mock.patch("certbot_nginx.configurator.logger") as mock_logger: @@ -586,22 +516,18 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(mock_logger.info.call_args[0][0], 'No matching insecure server blocks listening on port %s found.') - def test_no_double_redirect(self): - # Test that we don't also add the commented redirect if we've just added - # a redirect to that vhost this run + def test_double_redirect(self): + # Test that we add one redirect for each domain example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("example.com", "redirect") self.config.enhance("example.org", "redirect") - unexpected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' if ($scheme != "https") {'], - ['#', ' return 301 https://$host$request_uri;'], - ['#', ' } # managed by Certbot'] - ] + expected1 = UnspacedList(_redirect_block_for_domain("example.com"))[0] + expected2 = UnspacedList(_redirect_block_for_domain("example.org"))[0] + generated_conf = self.config.parser.parsed[example_conf] - for line in unexpected: - self.assertFalse(util.contains_at_depth(generated_conf, line, 2)) + self.assertTrue(util.contains_at_depth(generated_conf, expected1, 2)) + self.assertTrue(util.contains_at_depth(generated_conf, expected2, 2)) def test_staple_ocsp_bad_version(self): self.config.version = (1, 3, 1) @@ -763,7 +689,7 @@ class NginxConfiguratorTest(util.NginxTest): self.config.parser.load() - expected = ['return', '301', 'https://$host$request_uri'] + expected = UnspacedList(_redirect_block_for_domain("www.nomatch.com"))[0] generated_conf = self.config.parser.parsed[default_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index 92cb0e086..b30338b5b 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -162,17 +162,15 @@ class VirtualHostTest(unittest.TestCase): 'enabled: False']) self.assertEqual(stringified, str(self.vhost1)) - def test_has_redirect(self): - self.assertTrue(self.vhost1.has_redirect()) - self.assertTrue(self.vhost2.has_redirect()) - self.assertTrue(self.vhost3.has_redirect()) - self.assertFalse(self.vhost4.has_redirect()) - def test_contains_list(self): from certbot_nginx.obj import VirtualHost from certbot_nginx.obj import Addr - from certbot_nginx.configurator import REDIRECT_BLOCK, _test_block_from_block - test_needle = _test_block_from_block(REDIRECT_BLOCK) + from certbot_nginx.configurator import _test_block_from_block + test_block = [ + ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + ['\n'] + ] + test_needle = _test_block_from_block(test_block) test_haystack = [['listen', '80'], ['root', '/var/www/html'], ['index', 'index.html index.htm index.nginx-debian.html'], ['server_name', 'two.functorkitten.xyz'], ['listen', '443 ssl'], From a1aba5842e979379d98f3128d140d1270fb951c5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 24 Jan 2018 22:23:20 -0800 Subject: [PATCH 06/93] Fix --no-bootstrap on CentOS/RHEL 6 (#5476) * fix --no-bootstrap on RHEL6 * Add regression test --- letsencrypt-auto-source/letsencrypt-auto | 20 ++++++++++++------- .../letsencrypt-auto.template | 20 ++++++++++++------- .../tests/centos6_tests.sh | 8 ++++++++ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9c8ccf344..b5cac23e6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 96e5c2db0..2ce337002 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -300,13 +300,8 @@ DeterminePythonVersion() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -402,6 +397,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh index a0e96edf8..2c6dcf734 100644 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -69,5 +69,13 @@ fi echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present." echo "" +export VENV_PATH=$(mktemp -d) +"$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1 +if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1)" != 3 ]; then + echo "Python 3 wasn't used with --no-bootstrap!" + exit 1 +fi +unset VENV_PATH + # test using python3 pytest -v -s certbot/letsencrypt-auto-source/tests From a2239baa45de446ab246047171ad86e014bf580c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 24 Jan 2018 22:38:36 -0800 Subject: [PATCH 07/93] fix test_tests.sh (#5478) --- tests/letstest/scripts/test_tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index 4bed2dd3a..e6ab836b8 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -1,11 +1,13 @@ #!/bin/sh -xe +LE_AUTO="letsencrypt/letsencrypt-auto-source/letsencrypt-auto" +LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" MODULES="acme certbot certbot_apache certbot_nginx" VENV_NAME=venv # *-auto respects VENV_PATH -letsencrypt/certbot-auto --debug --os-packages-only --non-interactive -LE_AUTO_SUDO="" VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --no-bootstrap --non-interactive --version +$LE_AUTO --os-packages-only +LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version . $VENV_NAME/bin/activate # change to an empty directory to ensure CWD doesn't affect tests From 43bbaadd1154c0cd44eeb761bf33563953e793ae Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Jan 2018 15:29:38 -0800 Subject: [PATCH 08/93] Update certbot-auto and help (#5487) * Release 0.21.1 (cherry picked from commit ff60d70e68f7b4ddc60a61848190fbb6e55b5d2b) * Bump version to 0.22.0 --- certbot-auto | 46 ++++++++++-------- docs/cli-help.txt | 18 +++---- letsencrypt-auto | 46 ++++++++++-------- letsencrypt-auto-source/certbot-auto.asc | 16 +++--- letsencrypt-auto-source/letsencrypt-auto | 24 ++++----- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++----- 7 files changed, 92 insertions(+), 82 deletions(-) diff --git a/certbot-auto b/certbot-auto index 558c330b2..d3a5c23e5 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0" +LE_AUTO_VERSION="0.21.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -1190,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/docs/cli-help.txt b/docs/cli-help.txt index f7318f0b3..abebdb9c9 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,9 +107,9 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.21.0 (certbot; - Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags + "". (default: CertbotACMEClient/0.21.1 (certbot; + darwin 10.13.3) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags encoded in the user agent are: --duplicate, --force- renew, --allow-subset-of-names, -n, and whether any hooks are set. @@ -448,11 +448,9 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary. (default: - a2enmod) + Path to the Apache 'a2enmod' binary. (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary. (default: - a2dismod) + Path to the Apache 'a2dismod' binary. (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension. (default: -le- ssl.conf) @@ -466,13 +464,13 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration. (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for - you.(Only Ubuntu/Debian currently) (default: True) + you.(Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you.(Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 diff --git a/letsencrypt-auto b/letsencrypt-auto index 558c330b2..d3a5c23e5 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0" +LE_AUTO_VERSION="0.21.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -1190,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 9df8013a0..f28fd9893 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -Version: GnuPG v2 -iQEcBAABCAAGBQJaX+JUAAoJEE0XyZXNl3XyUCkH/jowI7yayXREoBUWpLuByd/n -e1wGLQjnZYkxv/AJGJ63G3QvwpzmIqo3r/6K4ARlUcdOnepZRDpF6jC4F5q9vBwW -AvUVU2B7e6mC6l/jXNepS8xowEwkQptQBDfnqh8TTeTb3rQTFod8X41skZ2633HL -RX4ditKaGMbcswMn6+5/juz0YK5ujVdVTcMeMcZKP2tvPJ9Y08YdpY6IdrM0Mfhn -IqssjM06CzsiYHeNOXfRY4vAPw4Oq/md3bf6ZpPCee1HPiDm0NvHtTemWBkPIehf -yy0U8JIDIZha4WKo3yifbZFL5Zf5czVkrtqQ3DBRcLrCFtBh2aTVsIMJkpW/wFo= -=d/hS +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlpqMlYACgkQTRfJlc2X +dfKHfQgAnZQJ34jFoVqEodT0EjvkFKZif4V/zXTsVwTHn107BcLCpH/9gjANrSo3 +JpvseH2q0odhOAZA4rZKH4Geh+5fsUl3Ew9YB28RXeyqEfCATUqPq6q+jAi55SLc +a064Ux5N7eOIh9gxvpDKBeSFD0eNB8IDtPQhUspr+WnoycawrJHNGawL8WIfrWY3 +0ZPF981iPCWCdN3woDP9wHA2QtBClAk2pQ1aMgdkK9r/QLO+DY92xmT/Uu4ik2jR +zv+QplsQLftjD+bRar5R9jiCWV5phPqrOF3ypMiU0K5bsnrZfGBzBcoEyfKuB+UR +F/j/631OC6yLRasr+xcL1gc+SCryfA== +=tkZT -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index b5cac23e6..8ff7944b5 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1196,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 2e52f03fdf9a1fbe2049d89f071a583bcf919976..8dd6837754301180039bdcd2192ec2ffa9911f1e 100644 GIT binary patch literal 256 zcmV+b0ssEOC2SB&#jbL|t)G85E<)sA{+|s~F5m$!7y+-fA{(45p3@Fk=k3Xi+RklE z=idnaoUY1-Gy6T(2&LYe?9T(@a*CNN5}h_$-t&E@K&w437B8*P8QflhULg&MO?mxD$RuiY1;Ik~L_O?oC&* zWM|7qv=mZh!YK!wmdDwpz@wj96nhJniJ_laAj^74?$6FZ`p5@HrirST)N_#L(FvXz GZCemMz Date: Wed, 22 Nov 2017 18:01:56 -0800 Subject: [PATCH 09/93] Add expiration date to skipped message --- certbot/renewal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/certbot/renewal.py b/certbot/renewal.py index 7d0240f73..024a815cc 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -425,7 +425,10 @@ def handle_renewal_request(config): main.renew_cert(lineage_config, plugins, renewal_candidate) renew_successes.append(renewal_candidate.fullchain) else: - renew_skipped.append(renewal_candidate.fullchain) + expiry = crypto_util.notAfter(renewal_candidate.version( + "cert", renewal_candidate.latest_common_version())) + renew_skipped.append("%s expires on %s" % (renewal_candidate.fullchain, + expiry.strftime("%Y-%m-%d"))) except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. logger.warning("Attempting to renew cert (%s) from %s produced an " From 45613fd31c3649ac5fb08a52153a8893f394af0b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 26 Jan 2018 16:02:19 -0800 Subject: [PATCH 10/93] update changelog for 0.21.1 (#5504) --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6f7be9fe..1369b0907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.21.1 - 2018-01-25 + +### Fixed + +* When creating an HTTP to HTTPS redirect in Nginx, we now ensure the Host + header of the request is set to an expected value before redirecting users to + the domain found in the header. The previous way Certbot configured Nginx + redirects was a potential security issue which you can read more about at + https://community.letsencrypt.org/t/security-issue-with-redirects-added-by-certbots-nginx-plugin/51493. +* Fixed a problem where Certbot's Apache plugin could fail HTTP-01 challenges + if basic authentication is configured for the domain you request a + certificate for. +* certbot-auto --no-bootstrap now properly tries to use Python 3.4 on RHEL 6 + based systems rather than Python 2.6. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/49?closed=1 + ## 0.21.0 - 2018-01-17 ### Added From 72b63ca5ac19cc8da0b1aba3a55dea85bca7a0eb Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 1 Feb 2018 13:14:43 -0800 Subject: [PATCH 11/93] Use "certificate" instead of "cert" in docs. --- docs/install.rst | 2 +- docs/using.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index a914586ff..c18c3cdbc 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -116,7 +116,7 @@ certbot-auto_ method, which enables you to use installer plugins that cover both of those hard topics. If you're still not convinced and have decided to use this method, -from the server that the domain you're requesting a cert for resolves +from the server that the domain you're requesting a certficate for resolves to, `install Docker`_, then issue the following command: .. code-block:: shell diff --git a/docs/using.rst b/docs/using.rst index ab4670052..e8f84e2d7 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -557,8 +557,8 @@ apologize for any inconvenience you encounter in integrating these commands into your individual environment. .. note:: ``certbot renew`` exit status will only be 1 if a renewal attempt failed. - This means ``certbot renew`` exit status will be 0 if no cert needs to be updated. - If you write a custom script and expect to run a command only after a cert was actually renewed + This means ``certbot renew`` exit status will be 0 if no certificate needs to be updated. + If you write a custom script and expect to run a command only after a certificate was actually renewed you will need to use the ``--post-hook`` since the exit status will be 0 both on successful renewal and when renewal is not necessary. From e085ff06a12a0be6033f38f092d42253c7b21515 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Mon, 5 Feb 2018 16:27:21 -0800 Subject: [PATCH 12/93] Update old issue link to point to letsencrypt community forums. (#5538) --- certbot/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index b735421f5..bc25da549 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -556,11 +556,11 @@ class Client(object): self.installer.rollback_checkpoints() self.installer.restart() except: - # TODO: suggest letshelp-letsencrypt here reporter.add_message( "An error occurred and we failed to restore your config and " - "restart your server. Please submit a bug report to " - "https://github.com/letsencrypt/letsencrypt", + "restart your server. Please post to " + "https://community.letsencrypt.org/c/server-config " + "with details about your configuration and this error you received.", reporter.HIGH_PRIORITY) raise reporter.add_message(success_msg, reporter.HIGH_PRIORITY) From 9baf75d6c887a9b69bd92c7bffe585f091c9b08e Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 6 Feb 2018 16:45:33 -0800 Subject: [PATCH 13/93] client.py changes for ACMEv2 (#5287) * Implement ACMEv2 signing of POST bodies. * Add account, and make acme_version explicit. * Remove separate NewAccount. * Rename to add v2. * Add terms_of_service_agreed. * Split out wrap_in_jws_v2 test. * Re-add too-many-public-methods. * Split Client into ClientBase / Client / ClientV2 * Use camelCase for newAccount. * Make acme_version optional parameter on .post(). This allows us to instantiate a ClientNetwork before knowing the version. * Add kid unconditionally. --- acme/acme/client.py | 324 ++++++++++++++++++++++++--------------- acme/acme/client_test.py | 74 ++++++--- acme/acme/messages.py | 1 + 3 files changed, 258 insertions(+), 141 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index dc5efbe86..5d1add5a3 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -39,39 +39,24 @@ DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class Client(object): # pylint: disable=too-many-instance-attributes - """ACME client. - - .. todo:: - Clean up raised error types hierarchy, document, and handle (wrap) - instances of `.DeserializationError` raised in `from_json()`. +class ClientBase(object): # pylint: disable=too-many-instance-attributes + """ACME client base object. :ivar messages.Directory directory: - :ivar key: `.JWK` (private) - :ivar alg: `.JWASignature` - :ivar bool verify_ssl: Verify SSL certificates? - :ivar .ClientNetwork net: Client network. Useful for testing. If not - supplied, it will be initialized using `key`, `alg` and - `verify_ssl`. - + :ivar .ClientNetwork net: Client network. + :ivar int acme_version: ACME protocol version. 1 or 2. """ - def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, - net=None): + def __init__(self, directory, net, acme_version): """Initialize. - :param directory: Directory Resource (`.messages.Directory`) or - URI from which the resource will be downloaded. - + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. + :param int acme_version: ACME protocol version. 1 or 2. """ - self.key = key - self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net - - if isinstance(directory, six.string_types): - self.directory = messages.Directory.from_json( - self.net.get(directory).json()) - else: - self.directory = directory + self.directory = directory + self.net = net + self.acme_version = acme_version @classmethod def _regr_from_response(cls, response, uri=None, terms_of_service=None): @@ -83,28 +68,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes uri=response.headers.get('Location', uri), terms_of_service=terms_of_service) - def register(self, new_reg=None): - """Register. - - :param .NewRegistration new_reg: - - :returns: Registration Resource. - :rtype: `.RegistrationResource` - - """ - new_reg = messages.NewRegistration() if new_reg is None else new_reg - assert isinstance(new_reg, messages.NewRegistration) - - response = self.net.post(self.directory[new_reg], new_reg) - # TODO: handle errors - assert response.status_code == http_client.CREATED - - # "Instance of 'Field' has no key/contact member" bug: - # pylint: disable=no-member - return self._regr_from_response(response) - def _send_recv_regr(self, regr, body): - response = self.net.post(regr.uri, body) + response = self.net.post(regr.uri, body, acme_version=self.acme_version) # TODO: Boulder returns httplib.ACCEPTED #assert response.status_code == httplib.OK @@ -153,21 +118,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ return self._send_recv_regr(regr, messages.UpdateRegistration()) - def agree_to_tos(self, regr): - """Agree to the terms-of-service. - - Agree to the terms-of-service in a Registration Resource. - - :param regr: Registration Resource. - :type regr: `.RegistrationResource` - - :returns: Updated Registration Resource. - :rtype: `.RegistrationResource` - - """ - return self.update_registration( - regr.update(body=regr.body.update(agreement=regr.terms_of_service))) - def _authzr_from_response(self, response, identifier, uri=None): authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), @@ -176,42 +126,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes raise errors.UnexpectedUpdate(authzr) return authzr - def request_challenges(self, identifier, new_authzr_uri=None): - """Request challenges. - - :param .messages.Identifier identifier: Identifier to be challenged. - :param str new_authzr_uri: Deprecated. Do not use. - - :returns: Authorization Resource. - :rtype: `.AuthorizationResource` - - """ - if new_authzr_uri is not None: - logger.debug("request_challenges with new_authzr_uri deprecated.") - new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz, new_authz) - # TODO: handle errors - assert response.status_code == http_client.CREATED - return self._authzr_from_response(response, identifier) - - def request_domain_challenges(self, domain, new_authzr_uri=None): - """Request challenges for domain names. - - This is simply a convenience function that wraps around - `request_challenges`, but works with domain names instead of - generic identifiers. See ``request_challenges`` for more - documentation. - - :param str domain: Domain name to be challenged. - :param str new_authzr_uri: Deprecated. Do not use. - - :returns: Authorization Resource. - :rtype: `.AuthorizationResource` - - """ - return self.request_challenges(messages.Identifier( - typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) - def answer_challenge(self, challb, response): """Answer challenge. @@ -227,7 +141,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self.net.post(challb.uri, response) + response = self.net.post(challb.uri, response, + acme_version=self.acme_version) try: authzr_uri = response.links['up']['url'] except KeyError: @@ -288,6 +203,133 @@ class Client(object): # pylint: disable=too-many-instance-attributes response, authzr.body.identifier, authzr.uri) return updated_authzr, response + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + response = self.net.post(self.directory[messages.Revocation], + messages.Revocation( + certificate=cert, + reason=rsn), + content_type=None, + acme_version=self.acme_version) + if response.status_code != http_client.OK: + raise errors.ClientError( + 'Successful revocation must return HTTP OK status') + +class Client(ClientBase): + """ACME client for a v1 API. + + .. todo:: + Clean up raised error types hierarchy, document, and handle (wrap) + instances of `.DeserializationError` raised in `from_json()`. + + :ivar messages.Directory directory: + :ivar key: `.JWK` (private) + :ivar alg: `.JWASignature` + :ivar bool verify_ssl: Verify SSL certificates? + :ivar .ClientNetwork net: Client network. Useful for testing. If not + supplied, it will be initialized using `key`, `alg` and + `verify_ssl`. + + """ + + def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, + net=None): + """Initialize. + + :param directory: Directory Resource (`.messages.Directory`) or + URI from which the resource will be downloaded. + + """ + # pylint: disable=too-many-arguments + self.key = key + self.net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) if net is None else net + + if isinstance(directory, six.string_types): + directory = messages.Directory.from_json( + self.net.get(directory).json()) + super(Client, self).__init__(directory=directory, + net=net, acme_version=1) + + def register(self, new_reg=None): + """Register. + + :param .NewRegistration new_reg: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` + + """ + new_reg = messages.NewRegistration() if new_reg is None else new_reg + response = self.net.post(self.directory[new_reg], new_reg, + acme_version=1) + # TODO: handle errors + assert response.status_code == http_client.CREATED + + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + return self._regr_from_response(response) + + def agree_to_tos(self, regr): + """Agree to the terms-of-service. + + Agree to the terms-of-service in a Registration Resource. + + :param regr: Registration Resource. + :type regr: `.RegistrationResource` + + :returns: Updated Registration Resource. + :rtype: `.RegistrationResource` + + """ + return self.update_registration( + regr.update(body=regr.body.update(agreement=regr.terms_of_service))) + + def request_challenges(self, identifier, new_authzr_uri=None): + """Request challenges. + + :param .messages.Identifier identifier: Identifier to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + """ + if new_authzr_uri is not None: + logger.debug("request_challenges with new_authzr_uri deprecated.") + new_authz = messages.NewAuthorization(identifier=identifier) + response = self.net.post(self.directory.new_authz, new_authz, + acme_version=1) + # TODO: handle errors + assert response.status_code == http_client.CREATED + return self._authzr_from_response(response, identifier) + + def request_domain_challenges(self, domain, new_authzr_uri=None): + """Request challenges for domain names. + + This is simply a convenience function that wraps around + `request_challenges`, but works with domain names instead of + generic identifiers. See ``request_challenges`` for more + documentation. + + :param str domain: Domain name to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + """ + return self.request_challenges(messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) + def request_issuance(self, csr, authzrs): """Request issuance. @@ -311,7 +353,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes self.directory.new_cert, req, content_type=content_type, - headers={'Accept': content_type}) + headers={'Accept': content_type}, + acme_version=1) cert_chain_uri = response.links.get('up', {}).get('url') @@ -481,37 +524,65 @@ class Client(object): # pylint: disable=too-many-instance-attributes "Recursion limit reached. Didn't get {0}".format(uri)) return chain - def revoke(self, cert, rsn): - """Revoke certificate. - :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in - `.ComparableX509` - :param int rsn: Reason code for certificate revocation. +class ClientV2(ClientBase): + """ACME client for a v2 API. - :raises .ClientError: If revocation is unsuccessful. + :ivar messages.Directory directory: + :ivar .ClientNetwork net: Client network. + """ + + def __init__(self, directory, net): + """Initialize. + + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. + """ + super(ClientV2, self).__init__(directory=directory, + net=net, acme_version=2) + + def new_account(self, new_account): + """Register. + + :param .NewRegistration new_account: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` """ - response = self.net.post(self.directory[messages.Revocation], - messages.Revocation( - certificate=cert, - reason=rsn), - content_type=None) - if response.status_code != http_client.OK: - raise errors.ClientError( - 'Successful revocation must return HTTP OK status') + response = self.net.post(self.directory['newAccount'], new_account, + acme_version=2) + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + return self._regr_from_response(response) class ClientNetwork(object): # pylint: disable=too-many-instance-attributes - """Client network.""" + """Wrapper around requests that signs POSTs for authentication. + + Also adds user agent, and handles Content-Type. + """ JSON_CONTENT_TYPE = 'application/json' JOSE_CONTENT_TYPE = 'application/jose+json' JSON_ERROR_CONTENT_TYPE = 'application/problem+json' REPLAY_NONCE_HEADER = 'Replay-Nonce' - def __init__(self, key, alg=jose.RS256, verify_ssl=True, + """Initialize. + + :param key: Account private key + :param messages.Registration account: Account object. Required if you are + planning to use .post() with acme_version=2. + :param josepy.JWASignature alg: Algoritm to use in signing JWS. + :param bool verify_ssl: Whether to verify certificates on SSL connections. + :param str user_agent: String to send as User-Agent header. + :param float timeout: Timeout for requests. + """ + def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True, user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT): + # pylint: disable=too-many-arguments self.key = key + self.account = account self.alg = alg self.verify_ssl = verify_ssl self._nonces = set() @@ -527,21 +598,29 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes except Exception: # pylint: disable=broad-except pass - def _wrap_in_jws(self, obj, nonce): + def _wrap_in_jws(self, obj, nonce, url, acme_version): """Wrap `JSONDeSerializable` object in JWS. .. todo:: Implement ``acmePath``. :param .JSONDeSerializable obj: + :param str url: The URL to which this object will be POSTed :param bytes nonce: :rtype: `.JWS` """ jobj = obj.json_dumps(indent=2).encode() logger.debug('JWS payload:\n%s', jobj) - return jws.JWS.sign( - payload=jobj, key=self.key, alg=self.alg, - nonce=nonce).json_dumps(indent=2) + kwargs = { + "alg": self.alg, + "nonce": nonce + } + if acme_version == 2: + kwargs["url"] = url + kwargs["kid"] = self.account["uri"] + kwargs["key"] = self.key + # pylint: disable=star-args + return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2) @classmethod def _check_response(cls, response, content_type=None): @@ -714,8 +793,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes else: raise - def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs): - data = self._wrap_in_jws(obj, self._get_nonce(url)) + def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, + acme_version=1, **kwargs): + data = self._wrap_in_jws(obj, self._get_nonce(url), url, acme_version) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) self._add_nonce(response) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 84620fc99..662c32942 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -104,6 +104,23 @@ class ClientTest(unittest.TestCase): self.assertEqual(self.regr, self.client.register(self.new_reg)) # TODO: test POST call arguments + def test_new_account_v2(self): + directory = messages.Directory({ + "newAccount": 'https://www.letsencrypt-demo.org/acme/new-account', + }) + from acme.client import ClientV2 + client = ClientV2(directory, self.net) + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.regr = messages.RegistrationResource( + body=messages.Registration( + contact=self.contact, key=KEY.public_key()), + uri='https://www.letsencrypt-demo.org/acme/reg/1') + + self.assertEqual(self.regr, client.new_account(self.regr)) + def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: # pylint: disable=no-member @@ -142,20 +159,23 @@ class ClientTest(unittest.TestCase): self.client.request_challenges(self.identifier) self.net.post.assert_called_once_with( self.directory.new_authz, - messages.NewAuthorization(identifier=self.identifier)) + messages.NewAuthorization(identifier=self.identifier), + acme_version=1) def test_request_challenges_deprecated_arg(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier, new_authzr_uri="hi") self.net.post.assert_called_once_with( self.directory.new_authz, - messages.NewAuthorization(identifier=self.identifier)) + messages.NewAuthorization(identifier=self.identifier), + acme_version=1) def test_request_challenges_custom_uri(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier) self.net.post.assert_called_once_with( - 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY) + 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY, + acme_version=1) def test_request_challenges_unexpected_update(self): self._prepare_response_for_request_challenges() @@ -417,7 +437,8 @@ class ClientTest(unittest.TestCase): def test_revoke(self): self.client.revoke(self.certr.body, self.rsn) self.net.post.assert_called_once_with( - self.directory[messages.Revocation], mock.ANY, content_type=None) + self.directory[messages.Revocation], mock.ANY, content_type=None, + acme_version=1) def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) @@ -432,9 +453,22 @@ class ClientTest(unittest.TestCase): self.certr, self.rsn) +class MockJSONDeSerializable(jose.JSONDeSerializable): + # pylint: disable=missing-docstring + def __init__(self, value): + self.value = value + + def to_partial_json(self): + return {'foo': self.value} + + @classmethod + def from_json(cls, value): + pass # pragma: no cover + class ClientNetworkTest(unittest.TestCase): """Tests for acme.client.ClientNetwork.""" + # pylint: disable=too-many-public-methods def setUp(self): self.verify_ssl = mock.MagicMock() @@ -453,25 +487,27 @@ class ClientNetworkTest(unittest.TestCase): self.assertTrue(self.net.verify_ssl is self.verify_ssl) def test_wrap_in_jws(self): - class MockJSONDeSerializable(jose.JSONDeSerializable): - # pylint: disable=missing-docstring - def __init__(self, value): - self.value = value - - def to_partial_json(self): - return {'foo': self.value} - - @classmethod - def from_json(cls, value): - pass # pragma: no cover - # pylint: disable=protected-access jws_dump = self.net._wrap_in_jws( - MockJSONDeSerializable('foo'), nonce=b'Tg') + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url", + acme_version=1) jws = acme_jws.JWS.json_loads(jws_dump) self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) self.assertEqual(jws.signature.combined.nonce, b'Tg') + def test_wrap_in_jws_v2(self): + self.net.account = {'uri': 'acct-uri'} + # pylint: disable=protected-access + jws_dump = self.net._wrap_in_jws( + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url", + acme_version=2) + jws = acme_jws.JWS.json_loads(jws_dump) + self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) + self.assertEqual(jws.signature.combined.nonce, b'Tg') + self.assertEqual(jws.signature.combined.kid, u'acct-uri') + self.assertEqual(jws.signature.combined.url, u'url') + + def test_check_response_not_ok_jobj_no_error(self): self.response.ok = False self.response.json.return_value = {} @@ -701,13 +737,13 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.assertEqual(self.checked_response, self.net.post( 'uri', self.obj, content_type=self.content_type)) self.net._wrap_in_jws.assert_called_once_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) self.available_nonces = [] self.assertRaises(errors.MissingNonce, self.net.post, 'uri', self.obj, content_type=self.content_type) self.net._wrap_in_jws.assert_called_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) def test_post_wrong_initial_nonce(self): # HEAD self.available_nonces = [b'f', jose.b64encode(b'good')] diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 6daf55094..98993c4e1 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -251,6 +251,7 @@ class Registration(ResourceBody): contact = jose.Field('contact', omitempty=True, default=()) agreement = jose.Field('agreement', omitempty=True) status = jose.Field('status', omitempty=True) + terms_of_service_agreed = jose.Field('terms-of-service-agreed', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' From 0416382633c7b37f05193c0bc42d10d03ac13b5e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Feb 2018 17:01:58 -0800 Subject: [PATCH 14/93] Update leauto_upgrades with tests from #5402. (#5407) --- .../letstest/scripts/test_leauto_upgrades.sh | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 51472f2e6..355fead2e 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -65,6 +65,7 @@ iQIDAQAB " if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then + RUN_PYTHON3_TESTS=1 if command -v python3; then echo "Didn't expect Python 3 to be installed!" exit 1 @@ -85,11 +86,25 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; exit 1 fi unset VENV_PATH - EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) - if ! ./cb-auto -v --debug --version -n 2>&1 | grep "$EXPECTED_VERSION" ; then - echo "Certbot didn't upgrade as expected!" - exit 1 - fi +fi + +if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then + echo "Had problems checking for updates!" + exit 1 +fi + +EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) +if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | grep "$EXPECTED_VERSION" ; then + echo upgrade appeared to fail + exit 1 +fi + +if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then + echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ + exit 1 +fi + +if [ "$RUN_PYTHON3_TESTS" = 1 ]; then if ! command -v python3; then echo "Python3 wasn't properly installed" exit 1 @@ -98,11 +113,7 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; echo "Python3 wasn't used in venv!" exit 1 fi -elif ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then - echo upgrade appeared to fail - exit 1 fi - echo upgrade appeared to be successful if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then From 530a9590e6b9a912162facdae58924f0e750d314 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 7 Feb 2018 14:08:03 -0800 Subject: [PATCH 15/93] Add sudo to certbot-auto instructions. (#5501) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index c144a4f74..a3e5d530a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -32,7 +32,7 @@ a new plugin is introduced. .. code-block:: shell cd certbot - ./certbot-auto --os-packages-only + sudo ./certbot-auto --os-packages-only ./tools/venv.sh Then in each shell where you're working on the client, do: From 4f0aeb12fa30d56c3926713901bc02cc53d6b9d1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Feb 2018 14:14:26 -0800 Subject: [PATCH 16/93] Add find-duplicative-certs docs (#5547) * add find-duplicative-certs docs * address review feedback --- certbot/cert_manager.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index da85ed783..4240a0523 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -121,7 +121,28 @@ def domains_for_certname(config, certname): return lineage.names() if lineage else None def find_duplicative_certs(config, domains): - """Find existing certs that duplicate the request.""" + """Find existing certs that match the given domain names. + + This function searches for certificates whose domains are equal to + the `domains` parameter and certificates whose domains are a subset + of the domains in the `domains` parameter. If multiple certificates + are found whose names are a subset of `domains`, the one whose names + are the largest subset of `domains` is returned. + + If multiple certificates' domains are an exact match or equally + sized subsets, which matching certificates are returned is + undefined. + + :param config: Configuration. + :type config: :class:`certbot.configuration.NamespaceConfig` + :param domains: List of domain names + :type domains: `list` of `str` + + :returns: lineages representing the identically matching cert and the + largest subset if they exist + :rtype: `tuple` of `storage.RenewableCert` or `None` + + """ def update_certs_for_domain_matches(candidate_lineage, rv): """Return cert as identical_names_cert if it matches, or subset_names_cert if it matches as subset From d6b247c002d9e5986098c44c771f2177a0823aae Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 9 Feb 2018 12:54:15 -0800 Subject: [PATCH 17/93] Set ClientNetwork.account after registering (#5558) --- acme/acme/client.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 5d1add5a3..77e89e535 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -95,6 +95,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes update = regr.body if update is None else update body = messages.UpdateRegistration(**dict(update)) updated_regr = self._send_recv_regr(regr, body=body) + self.net.account = updated_regr return updated_regr def deactivate_registration(self, regr): @@ -555,8 +556,9 @@ class ClientV2(ClientBase): acme_version=2) # "Instance of 'Field' has no key/contact member" bug: # pylint: disable=no-member - return self._regr_from_response(response) - + regr = self._regr_from_response(response) + self.net.account = regr + return regr class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Wrapper around requests that signs POSTs for authentication. @@ -571,8 +573,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Initialize. :param key: Account private key - :param messages.Registration account: Account object. Required if you are - planning to use .post() with acme_version=2. + :param messages.RegistrationResource account: Account object. Required if you are + planning to use .post() with acme_version=2 for anything other than creating a new + account; may be set later after registering. :param josepy.JWASignature alg: Algoritm to use in signing JWS. :param bool verify_ssl: Whether to verify certificates on SSL connections. :param str user_agent: String to send as User-Agent header. From 1f45832460550d0982676368870af9cac69592a5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 9 Feb 2018 16:41:05 -0800 Subject: [PATCH 18/93] Suggest people try the community forum. (#5561) --- ISSUE_TEMPLATE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 55780c0ac..03917f8ca 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,3 +1,9 @@ +If you're having trouble using Certbot and aren't sure you've found a bug or +request for a new feature, please first try asking for help at +https://community.letsencrypt.org/. There is a much larger community there of +people familiar with the project who will be able to more quickly answer your +questions. + ## My operating system is (include version): From abc4a27613f1e8196a52f75b051c05c3d2153a2a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 12 Feb 2018 14:07:33 -0800 Subject: [PATCH 19/93] [Docs] restore docs for ppl just using Certbot git master (#5420) - Dev / test cycles are one use case for the "running a local copy of the client" instructions, but simply running bleeding edge Certbot is another - So edit the docs to once again explain how to just run bleeding edge Certbot, without (say) always getting staging certs. --- docs/contributing.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index a3e5d530a..83b607e15 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -35,7 +35,10 @@ a new plugin is introduced. sudo ./certbot-auto --os-packages-only ./tools/venv.sh -Then in each shell where you're working on the client, do: +You can now run the copy of Certbot from git either by executing +``venv/bin/certbot``, or by activating the virtual environment. If you're +actively modifying and testing the code, you may want to run commands like this in +each shell where you're working: .. code-block:: shell @@ -43,7 +46,8 @@ Then in each shell where you're working on the client, do: export SERVER=https://acme-staging.api.letsencrypt.org/directory source tests/integration/_common.sh -After that, your shell will be using the virtual environment, and you run the +After that, your shell will be using the virtual environment, your copy of +Certbot will default to requesting test (staging) certificates, and you run the client by typing `certbot` or `certbot_test`. The latter is an alias that includes several flags useful for testing. For instance, it sets various output directories to point to /tmp/, and uses non-privileged ports for challenges, so From 789be8f9bc36241eee88849a6c496dc898185030 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 12 Feb 2018 14:55:41 -0800 Subject: [PATCH 20/93] Change "Attempting to parse" warning to info. (#5557) * Change "Attempting to parse" warning to info. This message shows up on every renewal run when the config was updated by a newer version of Certbot than the one being run. For instance, if a user has the certbot packages installed from PPA (currently 0.18.2), but runs certbot-auto once to try out the latest version (0.21.1), they will start getting this message via email every 12 hours. --- certbot/storage.py | 2 +- certbot/tests/storage_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot/storage.py b/certbot/storage.py index 67d2155ae..ed3922c58 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -417,7 +417,7 @@ class RenewableCert(object): conf_version = self.configuration.get("version") if (conf_version is not None and util.get_strict_version(conf_version) > CURRENT_VERSION): - logger.warning( + logger.info( "Attempting to parse the version %s renewal configuration " "file found at %s with version %s of Certbot. This might not " "work.", conf_version, config_filename, certbot.__version__) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 6c8f775e2..6c0970e72 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -158,8 +158,8 @@ class RenewableCertTests(BaseRenewableCertTest): with mock.patch("certbot.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) - self.assertTrue(mock_logger.warning.called) - self.assertTrue("version" in mock_logger.warning.call_args[0][0]) + self.assertTrue(mock_logger.info.called) + self.assertTrue("version" in mock_logger.info.call_args[0][0]) def test_consistent(self): # pylint: disable=too-many-statements,protected-access From 90664f196f992b20bdfdfb8914697e4456de7510 Mon Sep 17 00:00:00 2001 From: Eli Young Date: Mon, 12 Feb 2018 16:43:11 -0800 Subject: [PATCH 21/93] Remove autodocs for long-removed acme.other module (#5529) This module was removed in 22a9c7e3c2a811698ed7d63fae1cd43d0bd5d088. The autodocs are therefore unnecessary. Furthermore, they are starting to cause build failures for Fedora. --- acme/docs/api/other.rst | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 acme/docs/api/other.rst diff --git a/acme/docs/api/other.rst b/acme/docs/api/other.rst deleted file mode 100644 index eb27a5d53..000000000 --- a/acme/docs/api/other.rst +++ /dev/null @@ -1,5 +0,0 @@ -Other ACME objects ------------------- - -.. automodule:: acme.other - :members: From 932ecbb9c20bdc4e9d05c2e2bf78ea1b1608b67d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 13 Feb 2018 02:43:59 +0200 Subject: [PATCH 22/93] Fix test inconsistence in Apache plugin configurator_test (#5520) --- certbot-apache/certbot_apache/tests/configurator_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 530d75a92..7fbfcb617 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -474,7 +474,11 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(mock_add_dir.call_count, 3) self.assertTrue(mock_add_dir.called) self.assertEqual(mock_add_dir.call_args[0][1], "Listen") - self.assertEqual(mock_add_dir.call_args[0][2], ['1.2.3.4:8080']) + call_found = False + for mock_call in mock_add_dir.mock_calls: + if mock_call[1][2] == ['1.2.3.4:8080']: + call_found = True + self.assertTrue(call_found) def test_prepare_server_https(self): mock_enable = mock.Mock() From 49edf17cb77455f9f7b8227735939f693ceff303 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 13 Feb 2018 09:52:04 -0800 Subject: [PATCH 23/93] ignore .docker (#5477) --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index a01d2e1c7..4dff20caf 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ tests/letstest/venv/ # pytest cache .cache + +# docker files +.docker From ad0a99a1f5aa3cc28988685428bfdd475d48df58 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Tue, 13 Feb 2018 10:50:04 -0800 Subject: [PATCH 24/93] Proper webroot directory cleanup (#5453) * fix(webroot): clean up directories properly * fix(webroot): undo umask in finally --- certbot/plugins/util.py | 18 ++++++++++ certbot/plugins/util_test.py | 8 +++++ certbot/plugins/webroot.py | 63 +++++++++++++++++---------------- certbot/plugins/webroot_test.py | 31 ++++++++++++++++ 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index f0e2f4c5b..ad2257e1d 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,6 +6,24 @@ from certbot import util logger = logging.getLogger(__name__) +def get_prefixes(path): + """Retrieves all possible path prefixes of a path, in descending order + of length. For instance, + /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + :param str path: the path to break into prefixes + + :returns: all possible path prefixes of given path in descending order + :rtype: `list` of `str` + """ + prefix = path + prefixes = [] + while len(prefix) > 0: + prefixes.append(prefix) + prefix, _ = os.path.split(prefix) + # break once we hit '/' + if prefix == prefixes[-1]: + break + return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 947f24697..2c0e476ae 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,6 +5,14 @@ import unittest import mock +class GetPrefixTest(unittest.TestCase): + """Tests for certbot.plugins.get_prefixes.""" + def test_get_prefix(self): + from certbot.plugins.util import get_prefixes + self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes("/"), ["/"]) + self.assertEqual(get_prefixes("a"), ["a"]) + class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 714d83cce..15889a25c 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,6 +18,7 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common +from certbot.plugins import util logger = logging.getLogger(__name__) @@ -65,6 +66,8 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) + # stack of dirs successfully created by this authenticator + self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -161,27 +164,26 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) - try: - # This is coupled with the "umask" call above because - # os.makedirs's "mode" parameter may not always work: - # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python - os.makedirs(self.full_roots[name], 0o0755) - - # Set owner as parent directory if possible - try: - stat_path = os.stat(path) - os.chown(self.full_roots[name], stat_path.st_uid, - stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + stat_path = os.stat(path) + for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + try: + # This is coupled with the "umask" call above because + # os.mkdir's "mode" parameter may not always work: + # https://docs.python.org/3/library/os.html#os.mkdir + os.mkdir(prefix, 0o0755) + self._created_dirs.append(prefix) + # Set owner as parent directory if possible + try: + os.chown(prefix, stat_path.st_uid, stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}", name, exception) finally: os.umask(old_umask) @@ -217,16 +219,17 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - for root_path, achalls in six.iteritems(self.performed): - if not achalls: - try: - os.rmdir(root_path) - logger.debug("All challenges cleaned up, removing %s", - root_path) - except OSError as exc: - logger.info( - "Unable to clean up challenge directory %s", root_path) - logger.debug("Error was: %s", exc) + not_removed = [] + while len(self._created_dirs) > 0: + path = self._created_dirs.pop() + try: + os.rmdir(path) + except OSError as exc: + not_removed.insert(0, path) + logger.info("Challenge directory %s was not empty, didn't remove", path) + logger.debug("Error was: %s", exc) + self._created_dirs = not_removed + logger.debug("All challenges cleaned up") class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 36e2ffba6..59133f0aa 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,6 +36,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() + self.partial_root_challenge_path = os.path.join( + self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -199,6 +201,35 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) + self.assertFalse(os.path.exists(self.partial_root_challenge_path)) + + def test_perform_cleanup_existing_dirs(self): + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([self.achall]) + self.auth.cleanup([self.achall]) + + # Ensure we don't "clean up" directories that previously existed + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) + + def test_perform_cleanup_multiple_challenges(self): + bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"bingo"), "pending"), + domain="thing.com", account_key=KEY) + + bingo_validation_path = "YmluZ28" + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([bingo_achall, self.achall]) + + self.auth.cleanup([self.achall]) + self.assertFalse(os.path.exists(bingo_validation_path)) + self.assertTrue(os.path.exists(self.root_challenge_path)) + self.auth.cleanup([bingo_achall]) + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() From 9277710f6f984f6464ee375ea72d3a11b689853c Mon Sep 17 00:00:00 2001 From: sydneyli Date: Tue, 13 Feb 2018 11:15:08 -0800 Subject: [PATCH 25/93] Added install-only flag (#5531) --- letsencrypt-auto-source/letsencrypt-auto | 9 +++++++++ letsencrypt-auto-source/letsencrypt-auto.template | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8ff7944b5..c84499eab 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -1428,6 +1431,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2ce337002..365fc6752 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -587,6 +590,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else From ac464a58e5591f917ab66e2c30b2ba92cdc85e80 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 14 Feb 2018 18:16:20 +0200 Subject: [PATCH 26/93] Only add Include for TLS configuration if not already there (#5498) * Only add Include for TLS configuration if not already there * Add tests to prevent future regression --- certbot-apache/certbot_apache/configurator.py | 5 +++- .../certbot_apache/tests/configurator_test.py | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index b32eda921..4bb2cbebd 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1269,7 +1269,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "insert_cert_file_path") self.parser.add_dir(vh_path, "SSLCertificateKeyFile", "insert_key_file_path") - self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) + # Only include the TLS configuration if not already included + existing_inc = self.parser.find_dir("Include", self.mod_ssl_conf, vh_path) + if not existing_inc: + self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) def _add_servername_alias(self, target_name, vhost): vh_path = vhost.path diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 7fbfcb617..8f34d33d3 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -335,6 +335,33 @@ class MultipleVhostsTest(util.ApacheTest): "example/cert_chain.pem", "example/fullchain.pem") self.assertTrue(ssl_vhost.enabled) + def test_no_duplicate_include(self): + def mock_find_dir(directive, argument, _): + """Mock method for parser.find_dir""" + if directive == "Include" and argument.endswith("options-ssl-apache.conf"): + return ["/path/to/whatever"] + + mock_add = mock.MagicMock() + self.config.parser.add_dir = mock_add + self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access + tried_to_add = False + for a in mock_add.call_args_list: + if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf: + tried_to_add = True + # Include should be added, find_dir is not patched, and returns falsy + self.assertTrue(tried_to_add) + + self.config.parser.find_dir = mock_find_dir + mock_add.reset_mock() + + self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access + tried_to_add = [] + for a in mock_add.call_args_list: + tried_to_add.append(a[0][1] == "Include" and + a[0][2] == self.config.mod_ssl_conf) + # Include shouldn't be added, as patched find_dir "finds" existing one + self.assertFalse(any(tried_to_add)) + def test_deploy_cert(self): self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") From fbace69b5e8fed5ee32cb029478d6b11aea84842 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 14 Feb 2018 19:28:36 +0200 Subject: [PATCH 27/93] Fix install verb (#5536) * Fix install verb * Fix error message, tests and remove global pylint change * Fix boulder integration test keypath * Also use chain_path from lineage if not defined on CLI --- certbot/cli.py | 5 +- certbot/main.py | 41 ++++++++++++++-- certbot/tests/main_test.py | 90 +++++++++++++++++++++++++++++++++--- tests/boulder-integration.sh | 2 +- 4 files changed, 125 insertions(+), 13 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index dec7474f9..09dd71d13 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1278,14 +1278,13 @@ def _paths_parser(helpful): elif verb == "revoke": add(section, "--cert-path", type=read_file, required=True, help=cph) else: - add(section, "--cert-path", type=os.path.abspath, - help=cph, required=(verb == "install")) + add(section, "--cert-path", type=os.path.abspath, help=cph) section = "paths" if verb in ("install", "revoke"): section = verb # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", required=(verb == "install"), + add(section, "--key-path", type=((verb == "revoke" and read_file) or os.path.abspath), help="Path to private key for certificate installation " "or revocation (if account key is missing)") diff --git a/certbot/main.py b/certbot/main.py index 32dd69256..13234e0d2 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1,4 +1,5 @@ """Certbot main entry point.""" +# pylint: disable=too-many-lines from __future__ import print_function import functools import logging.handlers @@ -779,11 +780,45 @@ def install(config, plugins): except errors.PluginSelectionError as e: return str(e) - domains, _ = _find_domains_or_certname(config, installer) - le_client = _init_le_client(config, authenticator=None, installer=installer) - _install_cert(config, le_client, domains) + # If cert-path is defined, populate missing (ie. not overridden) values. + # Unfortunately this can't be done in argument parser, as certificate + # manager needs the access to renewal directory paths + if config.certname: + config = _populate_from_certname(config) + if config.key_path and config.cert_path: + _check_certificate_and_key(config) + domains, _ = _find_domains_or_certname(config, installer) + le_client = _init_le_client(config, authenticator=None, installer=installer) + _install_cert(config, le_client, domains) + else: + raise errors.ConfigurationError("Path to certificate or key was not defined. " + "If your certificate is managed by Certbot, please use --cert-name " + "to define which certificate you would like to install.") +def _populate_from_certname(config): + """Helper function for install to populate missing config values from lineage + defined by --cert-name.""" + lineage = cert_manager.lineage_for_certname(config, config.certname) + if not lineage: + return config + if not config.key_path: + config.namespace.key_path = lineage.key_path + if not config.cert_path: + config.namespace.cert_path = lineage.cert_path + if not config.chain_path: + config.namespace.chain_path = lineage.chain_path + if not config.fullchain_path: + config.namespace.fullchain_path = lineage.fullchain_path + return config + +def _check_certificate_and_key(config): + if not os.path.isfile(os.path.realpath(config.cert_path)): + raise errors.ConfigurationError("Error while reading certificate from path " + "{0}".format(config.cert_path)) + if not os.path.isfile(os.path.realpath(config.key_path)): + raise errors.ConfigurationError("Error while reading private key from path " + "{0}".format(config.key_path)) def plugins_cmd(config, plugins): """List server software plugins. diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index b1d58542f..7368e8513 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -592,11 +592,30 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met super(MainTest, self).tearDown() - def _call(self, args, stdout=None): - "Run the cli with output streams and actual client mocked out" - with mock.patch('certbot.main.client') as client: - ret, stdout, stderr = self._call_no_clientmock(args, stdout) - return ret, stdout, stderr, client + def _call(self, args, stdout=None, mockisfile=False): + """Run the cli with output streams, actual client and optionally + os.path.isfile() mocked out""" + + if mockisfile: + orig_open = os.path.isfile + def mock_isfile(fn, *args, **kwargs): + """Mock os.path.isfile()""" + if (fn.endswith("cert") or + fn.endswith("chain") or + fn.endswith("privkey")): + return True + else: + return orig_open(fn, *args, **kwargs) + + with mock.patch("os.path.isfile") as mock_if: + mock_if.side_effect = mock_isfile + with mock.patch('certbot.main.client') as client: + ret, stdout, stderr = self._call_no_clientmock(args, stdout) + return ret, stdout, stderr, client + else: + with mock.patch('certbot.main.client') as client: + ret, stdout, stderr = self._call_no_clientmock(args, stdout) + return ret, stdout, stderr, client def _call_no_clientmock(self, args, stdout=None): "Run the client with output streams mocked out" @@ -680,9 +699,68 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_selection(self, mock_pick_installer, _rec): self._call(['install', '--domains', 'foo.bar', '--cert-path', 'cert', - '--key-path', 'key', '--chain-path', 'chain']) + '--key-path', 'privkey', '--chain-path', 'chain'], mockisfile=True) self.assertEqual(mock_pick_installer.call_count, 1) + @mock.patch('certbot.main._install_cert') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_certname(self, _inst, _rec, mock_install): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install', '--cert-name', 'whatever'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/privkey") + + @mock.patch('certbot.main._install_cert') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_param_override(self, _inst, _rec, mock_install): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install', '--cert-name', 'whatever', + '--key-path', '/tmp/overriding_privkey'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.chain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/overriding_privkey") + + mock_install.reset() + + self._call(['install', '--cert-name', 'whatever', + '--cert-path', '/tmp/overriding_cert'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/overriding_cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/privkey") + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_param_error(self, _inst, _rec): + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--key-path', '/tmp/key_path']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--cert-path', '/tmp/key_path']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--cert-name', 'notfound', + '--key-path', 'invalid']) + @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.util.exe_exists') def test_configurator_selection(self, mock_exe_exists, unused_report): diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index e1aad4336..24d224cb0 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -251,7 +251,7 @@ openssl x509 -in "${root}/csr/chain.pem" -text common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ - --key-path "${root}/csr/key.pem" + --key-path "${root}/key.pem" CheckCertCount() { CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` From 99aec1394df3813646a2734405043aeada160a49 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Wed, 14 Feb 2018 12:09:17 -0800 Subject: [PATCH 28/93] Revert "Proper webroot directory cleanup (#5453)" (#5574) This reverts commit ad0a99a1f5aa3cc28988685428bfdd475d48df58. --- certbot/plugins/util.py | 18 ---------- certbot/plugins/util_test.py | 8 ----- certbot/plugins/webroot.py | 63 ++++++++++++++++----------------- certbot/plugins/webroot_test.py | 31 ---------------- 4 files changed, 30 insertions(+), 90 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index ad2257e1d..f0e2f4c5b 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,24 +6,6 @@ from certbot import util logger = logging.getLogger(__name__) -def get_prefixes(path): - """Retrieves all possible path prefixes of a path, in descending order - of length. For instance, - /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] - :param str path: the path to break into prefixes - - :returns: all possible path prefixes of given path in descending order - :rtype: `list` of `str` - """ - prefix = path - prefixes = [] - while len(prefix) > 0: - prefixes.append(prefix) - prefix, _ = os.path.split(prefix) - # break once we hit '/' - if prefix == prefixes[-1]: - break - return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 2c0e476ae..947f24697 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,14 +5,6 @@ import unittest import mock -class GetPrefixTest(unittest.TestCase): - """Tests for certbot.plugins.get_prefixes.""" - def test_get_prefix(self): - from certbot.plugins.util import get_prefixes - self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) - self.assertEqual(get_prefixes("/"), ["/"]) - self.assertEqual(get_prefixes("a"), ["a"]) - class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 15889a25c..714d83cce 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,7 +18,6 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common -from certbot.plugins import util logger = logging.getLogger(__name__) @@ -66,8 +65,6 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) - # stack of dirs successfully created by this authenticator - self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -164,26 +161,27 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) + try: - stat_path = os.stat(path) - for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): - try: - # This is coupled with the "umask" call above because - # os.mkdir's "mode" parameter may not always work: - # https://docs.python.org/3/library/os.html#os.mkdir - os.mkdir(prefix, 0o0755) - self._created_dirs.append(prefix) - # Set owner as parent directory if possible - try: - os.chown(prefix, stat_path.st_uid, stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + # This is coupled with the "umask" call above because + # os.makedirs's "mode" parameter may not always work: + # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python + os.makedirs(self.full_roots[name], 0o0755) + + # Set owner as parent directory if possible + try: + stat_path = os.stat(path) + os.chown(self.full_roots[name], stat_path.st_uid, + stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + + except OSError as exception: + if exception.errno != errno.EEXIST: + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}", name, exception) finally: os.umask(old_umask) @@ -219,17 +217,16 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - not_removed = [] - while len(self._created_dirs) > 0: - path = self._created_dirs.pop() - try: - os.rmdir(path) - except OSError as exc: - not_removed.insert(0, path) - logger.info("Challenge directory %s was not empty, didn't remove", path) - logger.debug("Error was: %s", exc) - self._created_dirs = not_removed - logger.debug("All challenges cleaned up") + for root_path, achalls in six.iteritems(self.performed): + if not achalls: + try: + os.rmdir(root_path) + logger.debug("All challenges cleaned up, removing %s", + root_path) + except OSError as exc: + logger.info( + "Unable to clean up challenge directory %s", root_path) + logger.debug("Error was: %s", exc) class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 59133f0aa..36e2ffba6 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,8 +36,6 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() - self.partial_root_challenge_path = os.path.join( - self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -201,35 +199,6 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) - self.assertFalse(os.path.exists(self.partial_root_challenge_path)) - - def test_perform_cleanup_existing_dirs(self): - os.mkdir(self.partial_root_challenge_path) - self.auth.prepare() - self.auth.perform([self.achall]) - self.auth.cleanup([self.achall]) - - # Ensure we don't "clean up" directories that previously existed - self.assertFalse(os.path.exists(self.validation_path)) - self.assertFalse(os.path.exists(self.root_challenge_path)) - - def test_perform_cleanup_multiple_challenges(self): - bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.HTTP01(token=b"bingo"), "pending"), - domain="thing.com", account_key=KEY) - - bingo_validation_path = "YmluZ28" - os.mkdir(self.partial_root_challenge_path) - self.auth.prepare() - self.auth.perform([bingo_achall, self.achall]) - - self.auth.cleanup([self.achall]) - self.assertFalse(os.path.exists(bingo_validation_path)) - self.assertTrue(os.path.exists(self.root_challenge_path)) - self.auth.cleanup([bingo_achall]) - self.assertFalse(os.path.exists(self.validation_path)) - self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() From 608875cd655e5719a56d178e4046539c98b91348 Mon Sep 17 00:00:00 2001 From: Sydney Li Date: Wed, 14 Feb 2018 15:14:24 -0800 Subject: [PATCH 29/93] Add test for skipped certs --- certbot/tests/main_test.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 45e5db1df..8a2ffd1ca 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -926,7 +926,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, - quiet_mode=False): + quiet_mode=False, expiry_date=datetime.datetime.now()): # pylint: disable=too-many-locals,too-many-arguments cert_path = test_util.vector_path('cert_512.pem') chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' @@ -959,7 +959,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mock_latest = mock.MagicMock() mock_latest.get_issuer.return_value = "Fake fake" mock_ssl.crypto.load_certificate.return_value = mock_latest - with mock.patch('certbot.main.renewal.crypto_util'): + with mock.patch('certbot.main.renewal.crypto_util') as mock_crypto_util: + mock_crypto_util.notAfter.return_value = expiry_date if not args: args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly'] if extra_args: @@ -1027,6 +1028,16 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, should_renew=True) + @mock.patch('certbot.renewal.should_renew') + def test_renew_skips_recent_certs(self, should_renew): + should_renew.return_value = False + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + expiry = datetime.datetime.now() + datetime.timedelta(days=90) + _, _, stdout = self._test_renewal_common(False, extra_args=None, should_renew=False, + args=['renew'], expiry_date=expiry) + self.assertTrue('No renewals were attempted.' in stdout.getvalue()) + self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue()) + def test_quiet_renew(self): test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run"] From 09b5927e6ac881359f6df9a1792a598fd94ad8d6 Mon Sep 17 00:00:00 2001 From: cclauss Date: Thu, 15 Feb 2018 20:07:35 +0100 Subject: [PATCH 30/93] from botocore.exceptions import ClientError (#5507) Fixes undefined name 'botocore' in flake8 testing of https://github.com/certbot/certbot $ __flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics__ ``` ./tests/letstest/multitester.py:144:12: F821 undefined name 'botocore' except botocore.exceptions.ClientError as e: ^ 1 F821 undefined name 'botocore' ``` --- tests/letstest/multitester.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index d9491939c..17740cde8 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -38,6 +38,7 @@ from multiprocessing import Manager import urllib2 import yaml import boto3 +from botocore.exceptions import ClientError import fabric from fabric.api import run, execute, local, env, sudo, cd, lcd from fabric.operations import get, put @@ -141,7 +142,7 @@ def make_instance(instance_name, # give instance a name try: new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) - except botocore.exceptions.ClientError as e: + except ClientError as e: if "InvalidInstanceID.NotFound" in str(e): # This seems to be ephemeral... retry time.sleep(1) From d5efefd979b7c660130b04e5c588d2f4a63a991c Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 15 Feb 2018 15:55:08 -0800 Subject: [PATCH 31/93] Re-land proper webroot directory cleanup (#5577) * fix(webroot): clean up directories properly * fix(webroot): undo umask in finally * Fix for MacOS --- certbot/plugins/util.py | 18 ++++++++++ certbot/plugins/util_test.py | 8 +++++ certbot/plugins/webroot.py | 63 +++++++++++++++++---------------- certbot/plugins/webroot_test.py | 31 ++++++++++++++++ 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index f0e2f4c5b..ad2257e1d 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,6 +6,24 @@ from certbot import util logger = logging.getLogger(__name__) +def get_prefixes(path): + """Retrieves all possible path prefixes of a path, in descending order + of length. For instance, + /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + :param str path: the path to break into prefixes + + :returns: all possible path prefixes of given path in descending order + :rtype: `list` of `str` + """ + prefix = path + prefixes = [] + while len(prefix) > 0: + prefixes.append(prefix) + prefix, _ = os.path.split(prefix) + # break once we hit '/' + if prefix == prefixes[-1]: + break + return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 947f24697..2c0e476ae 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,6 +5,14 @@ import unittest import mock +class GetPrefixTest(unittest.TestCase): + """Tests for certbot.plugins.get_prefixes.""" + def test_get_prefix(self): + from certbot.plugins.util import get_prefixes + self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes("/"), ["/"]) + self.assertEqual(get_prefixes("a"), ["a"]) + class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 714d83cce..6328b16ef 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,6 +18,7 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common +from certbot.plugins import util logger = logging.getLogger(__name__) @@ -65,6 +66,8 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) + # stack of dirs successfully created by this authenticator + self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -161,27 +164,26 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) - try: - # This is coupled with the "umask" call above because - # os.makedirs's "mode" parameter may not always work: - # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python - os.makedirs(self.full_roots[name], 0o0755) - - # Set owner as parent directory if possible - try: - stat_path = os.stat(path) - os.chown(self.full_roots[name], stat_path.st_uid, - stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + stat_path = os.stat(path) + for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + try: + # This is coupled with the "umask" call above because + # os.mkdir's "mode" parameter may not always work: + # https://docs.python.org/3/library/os.html#os.mkdir + os.mkdir(prefix, 0o0755) + self._created_dirs.append(prefix) + # Set owner as parent directory if possible + try: + os.chown(prefix, stat_path.st_uid, stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + except OSError as exception: + if exception.errno not in (errno.EEXIST, errno.EISDIR): + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}".format(name, exception)) finally: os.umask(old_umask) @@ -217,16 +219,17 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - for root_path, achalls in six.iteritems(self.performed): - if not achalls: - try: - os.rmdir(root_path) - logger.debug("All challenges cleaned up, removing %s", - root_path) - except OSError as exc: - logger.info( - "Unable to clean up challenge directory %s", root_path) - logger.debug("Error was: %s", exc) + not_removed = [] + while len(self._created_dirs) > 0: + path = self._created_dirs.pop() + try: + os.rmdir(path) + except OSError as exc: + not_removed.insert(0, path) + logger.info("Challenge directory %s was not empty, didn't remove", path) + logger.debug("Error was: %s", exc) + self._created_dirs = not_removed + logger.debug("All challenges cleaned up") class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 36e2ffba6..59133f0aa 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,6 +36,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() + self.partial_root_challenge_path = os.path.join( + self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -199,6 +201,35 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) + self.assertFalse(os.path.exists(self.partial_root_challenge_path)) + + def test_perform_cleanup_existing_dirs(self): + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([self.achall]) + self.auth.cleanup([self.achall]) + + # Ensure we don't "clean up" directories that previously existed + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) + + def test_perform_cleanup_multiple_challenges(self): + bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"bingo"), "pending"), + domain="thing.com", account_key=KEY) + + bingo_validation_path = "YmluZ28" + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([bingo_achall, self.achall]) + + self.auth.cleanup([self.achall]) + self.assertFalse(os.path.exists(bingo_validation_path)) + self.assertTrue(os.path.exists(self.root_challenge_path)) + self.auth.cleanup([bingo_achall]) + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() From d467a4ae95680220286c5ce7e6e840b48c8f74a5 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 15 Feb 2018 19:04:17 -0800 Subject: [PATCH 32/93] Add mechanism to detect acme version (#5554) Detects acme version by checking for newNonce field in the directory, since it's mandatory. Also updates ClientNetwork.account on register and update_registration. * add mechanism to detect acme version * update ClientNetwork.account comment * switch to MultiVersionClient object in acme * add shim methods * add returns * use backwards-compatible format and implement register * update to actual representation of tos v2 * add tos fields and pass through to v1 for partial updates * update tests * pass more tests * allow instance variable pass-through and lint * update certbot and tests to use new_account_and_tos method * remove --agree-tos test from main_test for now because we moved the callback into acme * add docstrings * use hasattr * all most review comments * use terms_of_service for both v1 and v2 * add tests for acme/client.py * tests for acme/messages.py --- acme/acme/client.py | 56 ++++++++++++ acme/acme/client_test.py | 171 ++++++++++++++++++++++++++++------- acme/acme/messages.py | 25 ++++- acme/acme/messages_test.py | 7 ++ certbot/client.py | 24 ++--- certbot/main.py | 11 ++- certbot/tests/client_test.py | 36 ++++---- certbot/tests/main_test.py | 17 ++-- certbot/tests/util.py | 2 +- 9 files changed, 262 insertions(+), 87 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 77e89e535..d843feaa7 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -560,6 +560,62 @@ class ClientV2(ClientBase): self.net.account = regr return regr +class BackwardsCompatibleClientV2(object): + """ACME client wrapper that tends towards V2-style calls, but + supports V1 servers. + + :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint + :ivar .ClientBase client: either Client or ClientV2 + """ + + def __init__(self, net, key, server): + directory = messages.Directory.from_json(net.get(server).json()) + self.acme_version = self._acme_version_from_directory(directory) + if self.acme_version == 1: + self.client = Client(directory, key=key, net=net) + else: + self.client = ClientV2(directory, net=net) + + def __getattr__(self, name): + if name in vars(self.client): + return getattr(self.client, name) + elif name in dir(ClientBase): + return getattr(self.client, name) + # temporary, for breaking changes into smaller pieces + elif name in dir(Client): + return getattr(self.client, name) + else: + raise AttributeError() + + def new_account_and_tos(self, regr, check_tos_cb=None): + """Combined register and agree_tos for V1, new_account for V2 + + :param .NewRegistration regr: + :param callable check_tos_cb: callback that raises an error if + the check does not work + """ + def _assess_tos(tos): + if check_tos_cb is not None: + check_tos_cb(tos) + if self.acme_version == 1: + regr = self.client.register(regr) + if regr.terms_of_service is not None: + _assess_tos(regr.terms_of_service) + return self.client.agree_to_tos(regr) + return regr + else: + if "terms_of_service" in self.client.directory.meta: + _assess_tos(self.client.directory.meta.terms_of_service) + regr = regr.update(terms_of_service_agreed=True) + return self.client.new_account(regr) + + def _acme_version_from_directory(self, directory): + if hasattr(directory, 'newNonce'): + return 2 + else: + return 1 + + class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Wrapper around requests that signs POSTs for authentication. diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 662c32942..eb03d9261 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -21,10 +21,25 @@ CERT_DER = test_util.load_vector('cert.der') KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) KEY2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) +DIRECTORY_V1 = messages.Directory({ + messages.NewRegistration: + 'https://www.letsencrypt-demo.org/acme/new-reg', + messages.Revocation: + 'https://www.letsencrypt-demo.org/acme/revoke-cert', + messages.NewAuthorization: + 'https://www.letsencrypt-demo.org/acme/new-authz', + messages.CertificateRequest: + 'https://www.letsencrypt-demo.org/acme/new-cert', +}) -class ClientTest(unittest.TestCase): - """Tests for acme.client.Client.""" - # pylint: disable=too-many-instance-attributes,too-many-public-methods +DIRECTORY_V2 = messages.Directory({ + 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', + 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce' +}) + + +class ClientTestBase(unittest.TestCase): + """Base for tests in acme.client.""" def setUp(self): self.response = mock.MagicMock( @@ -33,21 +48,6 @@ class ClientTest(unittest.TestCase): self.net.post.return_value = self.response self.net.get.return_value = self.response - self.directory = messages.Directory({ - messages.NewRegistration: - 'https://www.letsencrypt-demo.org/acme/new-reg', - messages.Revocation: - 'https://www.letsencrypt-demo.org/acme/revoke-cert', - messages.NewAuthorization: - 'https://www.letsencrypt-demo.org/acme/new-authz', - messages.CertificateRequest: - 'https://www.letsencrypt-demo.org/acme/new-cert', - }) - - from acme.client import Client - self.client = Client( - directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) - self.identifier = messages.Identifier( typ=messages.IDENTIFIER_FQDN, value='example.com') @@ -84,6 +84,124 @@ class ClientTest(unittest.TestCase): # Reason code for revocation self.rsn = 1 + +class BackwardsCompatibleClientV2Test(ClientTestBase): + """Tests for acme.client.BackwardsCompatibleClientV2.""" + + def _init(self): + uri = 'http://www.letsencrypt-demo.org/directory' + from acme.client import BackwardsCompatibleClientV2 + return BackwardsCompatibleClientV2(net=self.net, + key=KEY, server=uri) + + def test_init_downloads_directory(self): + uri = 'http://www.letsencrypt-demo.org/directory' + from acme.client import BackwardsCompatibleClientV2 + BackwardsCompatibleClientV2(net=self.net, + key=KEY, server=uri) + self.net.get.assert_called_once_with(uri) + + def test_init_acme_version(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + client = self._init() + self.assertEqual(client.acme_version, 1) + + self.response.json.return_value = DIRECTORY_V2.to_json() + client = self._init() + self.assertEqual(client.acme_version, 2) + + def test_forwarding(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + client = self._init() + self.assertEqual(client.directory, client.client.directory) + self.assertEqual(client.key, KEY) + # delete this line once we finish migrating to new API: + self.assertEqual(client.register, client.client.register) + self.assertEqual(client.update_registration, client.client.update_registration) + self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') + self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') + self.assertRaises(AttributeError, client.__getattr__, 'new_account') + + def test_new_account_and_tos(self): + # v2 no tos + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().new_account.assert_called_with(self.new_reg) + + # v2 tos good + with mock.patch('acme.client.ClientV2') as mock_client: + mock_client().directory.meta.__contains__.return_value = True + client = self._init() + client.new_account_and_tos(self.new_reg, lambda x: True) + mock_client().new_account.assert_called_with( + self.new_reg.update(terms_of_service_agreed=True)) + + # v2 tos bad + with mock.patch('acme.client.ClientV2') as mock_client: + mock_client().directory.meta.__contains__.return_value = True + client = self._init() + def _tos_cb(tos): + raise errors.Error + self.assertRaises(errors.Error, client.new_account_and_tos, + self.new_reg, _tos_cb) + mock_client().new_account.assert_not_called() + + # v1 yes tos + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + regr = mock.MagicMock(terms_of_service="TOS") + mock_client().register.return_value = regr + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().register.assert_called_once_with(self.new_reg) + mock_client().agree_to_tos.assert_called_once_with(regr) + + # v1 no tos + with mock.patch('acme.client.Client') as mock_client: + regr = mock.MagicMock(terms_of_service=None) + mock_client().register.return_value = regr + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().register.assert_called_once_with(self.new_reg) + mock_client().agree_to_tos.assert_not_called() + + +class ClientV2Test(ClientTestBase): + """Tests for acme.client.ClientV2.""" + # pylint: disable=too-many-instance-attributes,too-many-public-methods + + def setUp(self): + super(ClientV2Test, self).setUp() + from acme.client import ClientV2 + self.directory = DIRECTORY_V2 + self.client = ClientV2(directory=self.directory, net=self.net) + + def test_new_account_v2(self): + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.regr = messages.RegistrationResource( + body=messages.Registration( + contact=self.contact, key=KEY.public_key()), + uri='https://www.letsencrypt-demo.org/acme/reg/1') + + self.assertEqual(self.regr, self.client.new_account(self.regr)) + + +class ClientTest(ClientTestBase): + """Tests for acme.client.Client.""" + # pylint: disable=too-many-instance-attributes,too-many-public-methods + + def setUp(self): + super(ClientTest, self).setUp() + from acme.client import Client + self.directory = DIRECTORY_V1 + self.client = Client( + directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) + def test_init_downloads_directory(self): uri = 'http://www.letsencrypt-demo.org/directory' from acme.client import Client @@ -104,23 +222,6 @@ class ClientTest(unittest.TestCase): self.assertEqual(self.regr, self.client.register(self.new_reg)) # TODO: test POST call arguments - def test_new_account_v2(self): - directory = messages.Directory({ - "newAccount": 'https://www.letsencrypt-demo.org/acme/new-account', - }) - from acme.client import ClientV2 - client = ClientV2(directory, self.net) - self.response.status_code = http_client.CREATED - self.response.json.return_value = self.regr.body.to_json() - self.response.headers['Location'] = self.regr.uri - - self.regr = messages.RegistrationResource( - body=messages.Registration( - contact=self.contact, key=KEY.public_key()), - uri='https://www.letsencrypt-demo.org/acme/reg/1') - - self.assertEqual(self.regr, client.new_account(self.regr)) - def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: # pylint: disable=no-member diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 98993c4e1..21702f6a3 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -171,10 +171,31 @@ class Directory(jose.JSONDeSerializable): class Meta(jose.JSONObjectWithFields): """Directory Meta.""" - terms_of_service = jose.Field('terms-of-service', omitempty=True) + _terms_of_service = jose.Field('terms-of-service', omitempty=True) + _terms_of_service_v2 = jose.Field('termsOfService', omitempty=True) website = jose.Field('website', omitempty=True) caa_identities = jose.Field('caa-identities', omitempty=True) + def __init__(self, **kwargs): + kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items()) + # pylint: disable=star-args + super(Directory.Meta, self).__init__(**kwargs) + + @property + def terms_of_service(self): + """URL for the CA TOS""" + return self._terms_of_service or self._terms_of_service_v2 + + def __iter__(self): + # When iterating over fields, use the external name 'terms_of_service' instead of + # the internal '_terms_of_service'. + for name in super(Directory.Meta, self).__iter__(): + yield name[1:] if name == '_terms_of_service' else name + + def _internal_name(self, name): + return '_' + name if name == 'terms_of_service' else name + + @classmethod def _canon_key(cls, key): return getattr(key, 'resource_type', key) @@ -251,7 +272,7 @@ class Registration(ResourceBody): contact = jose.Field('contact', omitempty=True, default=()) agreement = jose.Field('agreement', omitempty=True) status = jose.Field('status', omitempty=True) - terms_of_service_agreed = jose.Field('terms-of-service-agreed', omitempty=True) + terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index fa885d3c2..4bc60a67b 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -165,6 +165,13 @@ class DirectoryTest(unittest.TestCase): from acme.messages import Directory Directory.from_json({'foo': 'bar'}) + def test_iter_meta(self): + result = False + for k in self.dir.meta: + if k == 'terms_of_service': + result = self.dir.meta[k] == 'https://example.com/acme/terms' + self.assertTrue(result) + class RegistrationTest(unittest.TestCase): """Tests for acme.messages.Registration.""" diff --git a/certbot/client.py b/certbot/client.py index bc25da549..67ee8f7fa 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -37,12 +37,12 @@ from certbot.plugins import selection as plugin_selection logger = logging.getLogger(__name__) -def acme_from_config_key(config, key): +def acme_from_config_key(config, key, regr=None): "Wrangle ACME client construction" # TODO: Allow for other alg types besides RS256 - net = acme_client.ClientNetwork(key, verify_ssl=(not config.no_verify_ssl), + net = acme_client.ClientNetwork(key, account=regr, verify_ssl=(not config.no_verify_ssl), user_agent=determine_user_agent(config)) - return acme_client.Client(config.server, key=key, net=net) + return acme_client.BackwardsCompatibleClientV2(net, key, config.server) def determine_user_agent(config): @@ -162,14 +162,7 @@ def register(config, account_storage, tos_cb=None): backend=default_backend()))) acme = acme_from_config_key(config, key) # TODO: add phone? - regr = perform_registration(acme, config) - - if regr.terms_of_service is not None: - if tos_cb is not None and not tos_cb(regr): - raise errors.Error( - "Registration cannot proceed without accepting " - "Terms of Service.") - regr = acme.agree_to_tos(regr) + regr = perform_registration(acme, config, tos_cb) acc = account.Account(regr, key) account.report_new_account(config) @@ -180,7 +173,7 @@ def register(config, account_storage, tos_cb=None): return acc, acme -def perform_registration(acme, config): +def perform_registration(acme, config, tos_cb): """ Actually register new account, trying repeatedly if there are email problems @@ -192,7 +185,8 @@ def perform_registration(acme, config): :rtype: `acme.messages.RegistrationResource` """ try: - return acme.register(messages.NewRegistration.from_data(email=config.email)) + return acme.new_account_and_tos(messages.NewRegistration.from_data(email=config.email), + tos_cb) except messages.Error as e: if e.code == "invalidEmail" or e.code == "invalidContact": if config.noninteractive_mode: @@ -202,7 +196,7 @@ def perform_registration(acme, config): raise errors.Error(msg) else: config.email = display_ops.get_email(invalid=True) - return perform_registration(acme, config) + return perform_registration(acme, config, tos_cb) else: raise @@ -232,7 +226,7 @@ class Client(object): # Initialize ACME if account is provided if acme is None and self.account is not None: - acme = acme_from_config_key(config, self.account.key) + acme = acme_from_config_key(config, self.account.key, self.account.regr) self.acme = acme if auth is not None: diff --git a/certbot/main.py b/certbot/main.py index 13234e0d2..ff3758985 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -496,17 +496,20 @@ def _determine_account(config): if config.email is None and not config.register_unsafely_without_email: config.email = display_ops.get_email() - def _tos_cb(regr): + def _tos_cb(terms_of_service): if config.tos: return True msg = ("Please read the Terms of Service at {0}. You " "must agree in order to register with the ACME " "server at {1}".format( - regr.terms_of_service, config.server)) + terms_of_service, config.server)) obj = zope.component.getUtility(interfaces.IDisplay) - return obj.yesno(msg, "Agree", "Cancel", + result = obj.yesno(msg, "Agree", "Cancel", cli_flag="--agree-tos", force_interactive=True) - + if not result: + raise errors.Error( + "Registration cannot proceed without accepting " + "Terms of Service.") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 204f46323..a9a87b80b 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -30,31 +30,27 @@ class RegisterTest(test_util.ConfigTestCase): self.config.register_unsafely_without_email = False self.config.email = "alias@example.com" self.account_storage = account.AccountMemoryStorage() - self.tos_cb = mock.MagicMock() def _call(self): from certbot.client import register - return register(self.config, self.account_storage, self.tos_cb) + tos_cb = mock.MagicMock() + return register(self.config, self.account_storage, tos_cb) def test_no_tos(self): - with mock.patch("certbot.client.acme_client.Client") as mock_client: - mock_client.register().terms_of_service = "http://tos" + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client.new_account_and_tos().terms_of_service = "http://tos" with mock.patch("certbot.eff.handle_subscription") as mock_handle: with mock.patch("certbot.account.report_new_account"): - self.tos_cb.return_value = False + mock_client().new_account_and_tos.side_effect = errors.Error self.assertRaises(errors.Error, self._call) self.assertFalse(mock_handle.called) - self.tos_cb.return_value = True + mock_client().new_account_and_tos.side_effect = None self._call() self.assertTrue(mock_handle.called) - self.tos_cb = None - self._call() - self.assertEqual(mock_handle.call_count, 2) - def test_it(self): - with mock.patch("certbot.client.acme_client.Client"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): with mock.patch("certbot.account.report_new_account"): with mock.patch("certbot.eff.handle_subscription"): self._call() @@ -66,9 +62,9 @@ class RegisterTest(test_util.ConfigTestCase): self.config.noninteractive_mode = False msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription") as mock_handle: - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self._call() self.assertEqual(mock_get_email.call_count, 1) self.assertTrue(mock_handle.called) @@ -79,9 +75,9 @@ class RegisterTest(test_util.ConfigTestCase): self.config.noninteractive_mode = True msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription"): - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(errors.Error, self._call) def test_needs_email(self): @@ -91,7 +87,7 @@ class RegisterTest(test_util.ConfigTestCase): @mock.patch("certbot.client.logger") def test_without_email(self, mock_logger): with mock.patch("certbot.eff.handle_subscription") as mock_handle: - with mock.patch("certbot.client.acme_client.Client"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): with mock.patch("certbot.account.report_new_account"): self.config.email = None self.config.register_unsafely_without_email = True @@ -104,9 +100,9 @@ class RegisterTest(test_util.ConfigTestCase): from acme import messages msg = "Test" mx_err = messages.Error(detail=msg, typ="malformed", title="title") - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription") as mock_handle: - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(messages.Error, self._call) self.assertFalse(mock_handle.called) @@ -122,7 +118,7 @@ class ClientTestCommon(test_util.ConfigTestCase): self.account = mock.MagicMock(**{"key.pem": KEY}) from certbot.client import Client - with mock.patch("certbot.client.acme_client.Client") as acme: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as acme: self.acme_client = acme self.acme = acme.return_value = mock.MagicMock() self.client = Client( @@ -140,7 +136,7 @@ class ClientTest(ClientTestCommon): self.eg_domains = ["example.com", "www.example.com"] def test_init_acme_verify_ssl(self): - net = self.acme_client.call_args[1]["net"] + net = self.acme_client.call_args[0][0] self.assertTrue(net.verify_ssl) def _mock_obtain_certificate(self): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index e84de54d0..518653a53 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -225,7 +225,7 @@ class RevokeTest(test_util.TempDirTestCase): 'cert_512.pem')) self.patches = [ - mock.patch('acme.client.Client', autospec=True), + mock.patch('acme.client.BackwardsCompatibleClientV2'), mock.patch('certbot.client.Client'), mock.patch('certbot.main._determine_account'), mock.patch('certbot.main.display_ops.success_revocation') @@ -267,7 +267,7 @@ class RevokeTest(test_util.TempDirTestCase): def test_revoke_with_reason(self, mock_acme_client, mock_delete_if_appropriate): mock_delete_if_appropriate.return_value = False - mock_revoke = mock_acme_client.Client().revoke + mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke expected = [] for reason, code in constants.REVOCATION_REASONS.items(): self._call("--reason " + reason) @@ -661,10 +661,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._cli_missing_flag(args, "specify a plugin") args.extend(['--standalone', '-d', 'eg.is']) self._cli_missing_flag(args, "register before running") - with mock.patch('certbot.main._get_and_save_cert'): - with mock.patch('certbot.main.client.acme_from_config_key'): - args.extend(['--email', 'io@io.is']) - self._cli_missing_flag(args, "--agree-tos") @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.main.client.acme_client.Client') @@ -693,7 +689,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met ua = "bandersnatch" args += ["--user-agent", ua] self._call_no_clientmock(args) - acme_net.assert_called_once_with(mock.ANY, verify_ssl=True, user_agent=ua) + acme_net.assert_called_once_with(mock.ANY, account=mock.ANY, verify_ssl=True, + user_agent=ua) @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') @@ -1352,11 +1349,11 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call_no_clientmock(['--cert-path', SS_CERT_PATH, '--key-path', RSA2048_KEY_PATH, '--server', server, 'revoke']) with open(RSA2048_KEY_PATH, 'rb') as f: - mock_acme_client.Client.assert_called_once_with( - server, key=jose.JWK.load(f.read()), net=mock.ANY) + mock_acme_client.BackwardsCompatibleClientV2.assert_called_once_with( + mock.ANY, jose.JWK.load(f.read()), server) with open(SS_CERT_PATH, 'rb') as f: cert = crypto_util.pyopenssl_load_certificate(f.read())[0] - mock_revoke = mock_acme_client.Client().revoke + mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke mock_revoke.assert_called_once_with( jose.ComparableX509(cert), mock.ANY) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index ddd4a1aec..60d8d6084 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -340,7 +340,7 @@ class ConfigTestCase(TempDirTestCase): self.config.cert_path = constants.CLI_DEFAULTS['auth_cert_path'] self.config.fullchain_path = constants.CLI_DEFAULTS['auth_chain_path'] self.config.chain_path = constants.CLI_DEFAULTS['auth_chain_path'] - self.config.server = "example.com" + self.config.server = "https://example.com" def lock_and_call(func, lock_path): """Grab a lock for lock_path and call func. From e48898a8c82e995470b909f13cc020ae7c32b5cd Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 14 Feb 2018 19:19:23 -0800 Subject: [PATCH 33/93] ACMEv2: Add Order support This adds two new classes in messages: Order and OrderResource. It also adds methods to ClientV2 to create orders, and poll orders then request issuance. The CSR is stored on the OrderResource so it can be carried along and submitted when it's time to finalize the order. --- acme/acme/client.py | 98 ++++++++++++++++++++++++++++++++++---- acme/acme/errors.py | 21 ++++++++ acme/acme/messages.py | 48 ++++++++++++++++++- acme/acme/messages_test.py | 18 ++++++- 4 files changed, 173 insertions(+), 12 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index d843feaa7..219667d53 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,6 +1,7 @@ """ACME client API.""" import base64 import collections +import cryptography import datetime from email.utils import parsedate_tz import heapq @@ -119,11 +120,11 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes """ return self._send_recv_regr(regr, messages.UpdateRegistration()) - def _authzr_from_response(self, response, identifier, uri=None): + def _authzr_from_response(self, response, identifier=None, uri=None): authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), uri=response.headers.get('Location', uri)) - if authzr.body.identifier != identifier: + if identifier is not None and authzr.body.identifier != identifier: raise errors.UnexpectedUpdate(authzr) return authzr @@ -233,8 +234,8 @@ class Client(ClientBase): instances of `.DeserializationError` raised in `from_json()`. :ivar messages.Directory directory: - :ivar key: `.JWK` (private) - :ivar alg: `.JWASignature` + :ivar key: `josepy.JWK` (private) + :ivar alg: `josepy.JWASignature` :ivar bool verify_ssl: Verify SSL certificates? :ivar .ClientNetwork net: Client network. Useful for testing. If not supplied, it will be initialized using `key`, `alg` and @@ -550,7 +551,6 @@ class ClientV2(ClientBase): :returns: Registration Resource. :rtype: `.RegistrationResource` - """ response = self.net.post(self.directory['newAccount'], new_account, acme_version=2) @@ -560,6 +560,84 @@ class ClientV2(ClientBase): self.net.account = regr return regr + def new_order(self, csr_pem): + """Request a new Order object from the server. + + :param str csr_pem: A CSR in PEM format. + + :returns: The newly created order. + :rtype: OrderResource + """ + csr = cryptography.x509.load_pem_x509_csr(csr_pem, + cryptography.hazmat.backends.default_backend()) + san_extension = next(ext for ext in csr.extensions + if ext.oid == cryptography.x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) + dnsNames = san_extension.value.get_values_for_type(cryptography.x509.DNSName) + + identifiers = [] + for name in dnsNames: + identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN, + value=name)) + order = messages.NewOrder(identifiers=identifiers) + response = self.net.post(self.directory['newOrder'], order) + body = messages.Order.from_json(response.json()) + authorizations = [] + for url in body.authorizations: + authorizations.append(self._authzr_from_response(self.net.get(url))) + return messages.OrderResource( + body=body, + uri=response.headers.get('Location', uri), + fullchain_pem=fullchain_pem, + authorizations=authorizations, + csr_pem=csr_pem) + + def poll_and_finalize(self, orderr, deadline=None): + if deadline is None: + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.poll_authorizations(orderr, deadline) + return self.finalize_order(orderr, deadline) + + def poll_authorizations(self, orderr, deadline): + """Poll Order Resource for status.""" + responses = [] + for url in orderr.body.authorizations: + while datetime.datetime.now() < deadline: + authzr = self._authzr_from_response(self.net.get(url), uri=url) + if authzr.body.status != messages.STATUS_PENDING: + responses.append(authzr) + break + time.sleep(1) + # If we didn't get a response for every authorization, we fell through + # the bottom of the loop due to hitting the deadline. + if len(responses) > orderr.body.authorizations: + raise TimeoutError() + failed = [] + for authzr in responses: + if authzr.body.status != messages.STATUS_VALID: + for chall in authzr.body.challenges: + if chall.error != None: + failed.append(authzr) + if len(failed) > 0: + raise ValidationError(failed) + return orderr.update(authorizations=responses) + + def finalize_order(self, orderr, deadline): + csr = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) + wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) + self.net.post(latest.body.finalize, wrapped_csr) + while datetime.datetime.now() < deadline: + time.sleep(1) + response = self.net.get(orderr.uri) + body = messages.Order.from_json(response.json()) + if body.error is not None: + raise IssuanceError(body.error) + if body.certificate is not None: + certificate_response = self.net.get(body.certificate).text + return orderr.update(fullchain_pem=certificate_response) + raise TimeoutError() + + class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but supports V1 servers. @@ -628,10 +706,10 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Initialize. - :param key: Account private key + :param josepy.JWK key: Account private key :param messages.RegistrationResource account: Account object. Required if you are - planning to use .post() with acme_version=2 for anything other than creating a new - account; may be set later after registering. + planning to use .post() with acme_version=2 for anything other than + creating a new account; may be set later after registering. :param josepy.JWASignature alg: Algoritm to use in signing JWS. :param bool verify_ssl: Whether to verify certificates on SSL connections. :param str user_agent: String to send as User-Agent header. @@ -662,10 +740,10 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes .. todo:: Implement ``acmePath``. - :param .JSONDeSerializable obj: + :param josepy.JSONDeSerializable obj: :param str url: The URL to which this object will be POSTed :param bytes nonce: - :rtype: `.JWS` + :rtype: `josepy.JWS` """ jobj = obj.json_dumps(indent=2).encode() diff --git a/acme/acme/errors.py b/acme/acme/errors.py index de5f9d1f4..624ccb3d9 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -83,6 +83,27 @@ class PollError(ClientError): return '{0}(exhausted={1!r}, updated={2!r})'.format( self.__class__.__name__, self.exhausted, self.updated) +class ValidationError(Error): + """Error for authorization failures. Contains a list of authorization + resources, each of which is invalid and should have an error field. + """ + def __init__(self, failed_authzrs): + self.failed_authzrs = failed_authzrs + super(ClientError, self).__init__() + +class TimeoutError(Error): + """Error for when polling an authorization or an order times out.""" + +class IssuanceError(Error): + """Error sent by the server after requesting issuance of a certificate.""" + + def __init__(self, error): + """Initialize. + + :param messages.Error error: The error provided by the server. + """ + self.error = error + class ConflictError(ClientError): """Error for when the server returns a 409 (Conflict) HTTP status. diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 21702f6a3..418bcead9 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -174,7 +174,7 @@ class Directory(jose.JSONDeSerializable): _terms_of_service = jose.Field('terms-of-service', omitempty=True) _terms_of_service_v2 = jose.Field('termsOfService', omitempty=True) website = jose.Field('website', omitempty=True) - caa_identities = jose.Field('caa-identities', omitempty=True) + caa_identities = jose.Field('caaIdentities', omitempty=True) def __init__(self, **kwargs): kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items()) @@ -504,3 +504,49 @@ class Revocation(jose.JSONObjectWithFields): certificate = jose.Field( 'certificate', decoder=jose.decode_cert, encoder=jose.encode_cert) reason = jose.Field('reason') + + +class Order(ResourceBody): + """Order Resource Body. + + .. note:: Parsing of identifiers on response doesn't work right now; to make + it work we would need to set up the equivalent of Identifier.from_json, but + for a list. + :ivar list of .Identifier: List of identifiers for the certificate. + :ivar acme.messages.Status status: + :ivar list of str authorizations: URLs of authorizations. + :ivar str certificate: URL to download certificate as a fullchain PEM. + :ivar str finalize: URL to POST to to request issuance once all + authorizations have "valid" status. + :ivar datetime.datetime expires: When the order expires. + :ivar .Error error: Any error that occurred during finalization, if applicable. + """ + identifiers = jose.Field('identifiers', omitempty=True) + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) + authorizations = jose.Field('authorizations', omitempty=True) + certificate = jose.Field('certificate', omitempty=True) + finalize = jose.Field('finalize', omitempty=True) + expires = fields.RFC3339Field('expires', omitempty=True) + error = jose.Field('error', omitempty=True, decoder=Error.from_json) + +class OrderResource(ResourceWithURI): + """Order Resource. + + :ivar acme.messages.Order body: + :ivar str csr_pem: The CSR this Order will be finalized with. + :ivar list of acme.messages.AuthorizationResource authorizations: + Fully-fetched AuthorizationResource objects. + :ivar str fullchain_pem: The fetched contents of the certificate URL + produced once the order was finalized, if it's present. + """ + body = jose.Field('body', decoder=Order.from_json) + csr_pem = jose.Field('csr_pem', omitempty=True) + authorizations = jose.Field('authorizations') + fullchain_pem = jose.Field('fullchain_pem', omitempty=True) + +@Directory.register +class NewOrder(Order): + """New order.""" + resource_type = 'new-order' + resource = fields.Resource(resource_type) diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 4bc60a67b..64bc81efd 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -157,7 +157,7 @@ class DirectoryTest(unittest.TestCase): 'meta': { 'terms-of-service': 'https://example.com/acme/terms', 'website': 'https://www.example.com/', - 'caa-identities': ['example.com'], + 'caaIdentities': ['example.com'], }, }) @@ -408,5 +408,21 @@ class RevocationTest(unittest.TestCase): hash(Revocation.from_json(self.rev.to_json())) +class OrderResourceTest(unittest.TestCase): + """Tests for acme.messages.OrderResource.""" + + def setUp(self): + from acme.messages import OrderResource + self.regr = OrderResource( + body=mock.sentinel.body, uri=mock.sentinel.uri) + + def test_to_partial_json(self): + self.assertEqual(self.regr.to_json(), { + 'body': mock.sentinel.body, + 'uri': mock.sentinel.uri, + 'authorizations': None, + }) + + if __name__ == '__main__': unittest.main() # pragma: no cover From 70a75ebe9dffb7d59e701e2dcb04ce23a1bcc0fc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Feb 2018 19:40:14 -0800 Subject: [PATCH 34/93] Add tests and fix minor bugs in Order support * delint * refactor client tests * Add test for new order and fix identifiers parsing. * Add poll_and_finalize test * Test and fix poll_authorizations timeout * Add test_failed_authorizations * Add test_poll_authorizations_success * Test and fix finalize_order success * add test_finalize_order_timeout * add test_finalize_order_error * test sleep code --- acme/acme/client.py | 38 ++++++-- acme/acme/client_test.py | 181 +++++++++++++++++++++++++++++++-------- acme/acme/errors.py | 3 +- acme/acme/messages.py | 7 +- 4 files changed, 180 insertions(+), 49 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 219667d53..bc93ca06f 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -586,12 +586,23 @@ class ClientV2(ClientBase): authorizations.append(self._authzr_from_response(self.net.get(url))) return messages.OrderResource( body=body, - uri=response.headers.get('Location', uri), - fullchain_pem=fullchain_pem, + uri=response.headers.get('Location'), authorizations=authorizations, csr_pem=csr_pem) def poll_and_finalize(self, orderr, deadline=None): + """Poll authorizations and finalize the order. + + If no deadline is provided, this method will timeout after 90 + seconds. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ if deadline is None: deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) orderr = self.poll_authorizations(orderr, deadline) @@ -609,8 +620,8 @@ class ClientV2(ClientBase): time.sleep(1) # If we didn't get a response for every authorization, we fell through # the bottom of the loop due to hitting the deadline. - if len(responses) > orderr.body.authorizations: - raise TimeoutError() + if len(responses) < len(orderr.body.authorizations): + raise errors.TimeoutError() failed = [] for authzr in responses: if authzr.body.status != messages.STATUS_VALID: @@ -618,24 +629,33 @@ class ClientV2(ClientBase): if chall.error != None: failed.append(authzr) if len(failed) > 0: - raise ValidationError(failed) + raise errors.ValidationError(failed) return orderr.update(authorizations=responses) def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) - self.net.post(latest.body.finalize, wrapped_csr) + self.net.post(orderr.body.finalize, wrapped_csr) while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) body = messages.Order.from_json(response.json()) if body.error is not None: - raise IssuanceError(body.error) + raise errors.IssuanceError(body.error) if body.certificate is not None: certificate_response = self.net.get(body.certificate).text - return orderr.update(fullchain_pem=certificate_response) - raise TimeoutError() + return orderr.update(body=body, fullchain_pem=certificate_response) + raise errors.TimeoutError() class BackwardsCompatibleClientV2(object): diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index eb03d9261..11516c02f 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -1,4 +1,5 @@ """Tests for acme.client.""" +import copy import datetime import json import unittest @@ -18,6 +19,8 @@ from acme import test_util CERT_DER = test_util.load_vector('cert.der') +CERT_SAN_PEM = test_util.load_vector('cert-san.pem') +CSR_SAN_PEM = test_util.load_vector('csr-san.pem') KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) KEY2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) @@ -34,7 +37,8 @@ DIRECTORY_V1 = messages.Directory({ DIRECTORY_V2 = messages.Directory({ 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', - 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce' + 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce', + 'newOrder': 'https://www.letsencrypt-demo.org/acme/new-order', }) @@ -57,8 +61,7 @@ class ClientTestBase(unittest.TestCase): contact=self.contact, key=KEY.public_key()) self.new_reg = messages.NewRegistration(**dict(reg)) self.regr = messages.RegistrationResource( - body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1', - terms_of_service='https://www.letsencrypt-demo.org/tos') + body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') # Authorization authzr_uri = 'https://www.letsencrypt-demo.org/acme/authz/1' @@ -75,15 +78,6 @@ class ClientTestBase(unittest.TestCase): self.authzr = messages.AuthorizationResource( body=self.authz, uri=authzr_uri) - # Request issuance - self.certr = messages.CertificateResource( - body=messages_test.CERT, authzrs=(self.authzr,), - uri='https://www.letsencrypt-demo.org/acme/cert/1', - cert_chain_uri='https://www.letsencrypt-demo.org/ca') - - # Reason code for revocation - self.rsn = 1 - class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" @@ -168,37 +162,29 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().agree_to_tos.assert_not_called() -class ClientV2Test(ClientTestBase): - """Tests for acme.client.ClientV2.""" - # pylint: disable=too-many-instance-attributes,too-many-public-methods - - def setUp(self): - super(ClientV2Test, self).setUp() - from acme.client import ClientV2 - self.directory = DIRECTORY_V2 - self.client = ClientV2(directory=self.directory, net=self.net) - - def test_new_account_v2(self): - self.response.status_code = http_client.CREATED - self.response.json.return_value = self.regr.body.to_json() - self.response.headers['Location'] = self.regr.uri - - self.regr = messages.RegistrationResource( - body=messages.Registration( - contact=self.contact, key=KEY.public_key()), - uri='https://www.letsencrypt-demo.org/acme/reg/1') - - self.assertEqual(self.regr, self.client.new_account(self.regr)) - - class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" # pylint: disable=too-many-instance-attributes,too-many-public-methods def setUp(self): super(ClientTest, self).setUp() - from acme.client import Client + self.directory = DIRECTORY_V1 + + # Registration + self.regr = self.regr.update( + terms_of_service='https://www.letsencrypt-demo.org/tos') + + # Request issuance + self.certr = messages.CertificateResource( + body=messages_test.CERT, authzrs=(self.authzr,), + uri='https://www.letsencrypt-demo.org/acme/cert/1', + cert_chain_uri='https://www.letsencrypt-demo.org/ca') + + # Reason code for revocation + self.rsn = 1 + + from acme.client import Client self.client = Client( directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) @@ -554,6 +540,129 @@ class ClientTest(ClientTestBase): self.certr, self.rsn) +class ClientV2Test(ClientTestBase): + """Tests for acme.client.ClientV2.""" + + def setUp(self): + super(ClientV2Test, self).setUp() + + self.directory = DIRECTORY_V2 + + from acme.client import ClientV2 + self.client = ClientV2(self.directory, self.net) + + self.new_reg = self.new_reg.update(terms_of_service_agreed=True) + + self.authzr_uri2 = 'https://www.letsencrypt-demo.org/acme/authz/2' + self.authz2 = self.authz.update(identifier=messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value='www.example.com'), + status=messages.STATUS_PENDING) + self.authzr2 = messages.AuthorizationResource( + body=self.authz2, uri=self.authzr_uri2) + + self.order = messages.Order( + identifiers=(self.authz.identifier, self.authz2.identifier), + status=messages.STATUS_PENDING, + authorizations=(self.authzr.uri, self.authzr_uri2), + finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') + self.orderr = messages.OrderResource( + body=self.order, + uri='https://www.letsencrypt-demo.org/acme/acct/1/order/1', + authorizations=[self.authzr, self.authzr2], csr_pem=CSR_SAN_PEM) + + def test_new_account(self): + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.assertEqual(self.regr, self.client.new_account(self.new_reg)) + + def test_new_order(self): + order_response = copy.deepcopy(self.response) + order_response.status_code = http_client.CREATED + order_response.json.return_value = self.order.to_json() + order_response.headers['Location'] = self.orderr.uri + self.net.post.return_value = order_response + + authz_response = copy.deepcopy(self.response) + authz_response.json.return_value = self.authz.to_json() + authz_response.headers['Location'] = self.authzr.uri + authz_response2 = self.response + authz_response2.json.return_value = self.authz2.to_json() + authz_response2.headers['Location'] = self.authzr2.uri + self.net.get.side_effect = (authz_response, authz_response2) + + self.assertEqual(self.client.new_order(CSR_SAN_PEM), self.orderr) + + @mock.patch('acme.client.datetime') + def test_poll_and_finalize(self, mock_datetime): + mock_datetime.datetime.now.return_value = datetime.datetime(2018, 2, 15) + mock_datetime.timedelta = datetime.timedelta + expected_deadline = mock_datetime.datetime.now() + datetime.timedelta(seconds=90) + + self.client.poll_authorizations = mock.Mock(return_value=self.orderr) + self.client.finalize_order = mock.Mock(return_value=self.orderr) + + self.assertEqual(self.client.poll_and_finalize(self.orderr), self.orderr) + self.client.poll_authorizations.assert_called_once_with(self.orderr, expected_deadline) + self.client.finalize_order.assert_called_once_with(self.orderr, expected_deadline) + + @mock.patch('acme.client.datetime') + def test_poll_authorizations_timeout(self, mock_datetime): + now_side_effect = [datetime.datetime(2018, 2, 15), + datetime.datetime(2018, 2, 16), + datetime.datetime(2018, 2, 17)] + mock_datetime.datetime.now.side_effect = now_side_effect + self.response.json.side_effect = [ + self.authz.to_json(), self.authz2.to_json(), self.authz2.to_json()] + + self.assertRaises( + errors.TimeoutError, self.client.poll_authorizations, self.orderr, now_side_effect[1]) + + def test_poll_authorizations_failure(self): + deadline = datetime.datetime(9999, 9, 9) + challb = self.challr.body.update(status=messages.STATUS_INVALID, + error=messages.Error.with_code('unauthorized')) + authz = self.authz.update(status=messages.STATUS_INVALID, challenges=(challb,)) + self.response.json.return_value = authz.to_json() + + self.assertRaises( + errors.ValidationError, self.client.poll_authorizations, self.orderr, deadline) + + def test_poll_authorizations_success(self): + deadline = datetime.datetime(9999, 9, 9) + updated_authz2 = self.authz2.update(status=messages.STATUS_VALID) + updated_authzr2 = messages.AuthorizationResource( + body=updated_authz2, uri=self.authzr_uri2) + updated_orderr = self.orderr.update(authorizations=[self.authzr, updated_authzr2]) + + self.response.json.side_effect = ( + self.authz.to_json(), self.authz2.to_json(), updated_authz2.to_json()) + self.assertEqual(self.client.poll_authorizations(self.orderr, deadline), updated_orderr) + + def test_finalize_order_success(self): + updated_order = self.order.update( + certificate='https://www.letsencrypt-demo.org/acme/cert/') + updated_orderr = self.orderr.update(body=updated_order, fullchain_pem=CERT_SAN_PEM) + + self.response.json.return_value = updated_order.to_json() + self.response.text = CERT_SAN_PEM + + deadline = datetime.datetime(9999, 9, 9) + self.assertEqual(self.client.finalize_order(self.orderr, deadline), updated_orderr) + + def test_finalize_order_error(self): + updated_order = self.order.update(error=messages.Error.with_code('unauthorized')) + self.response.json.return_value = updated_order.to_json() + + deadline = datetime.datetime(9999, 9, 9) + self.assertRaises(errors.IssuanceError, self.client.finalize_order, self.orderr, deadline) + + def test_finalize_order_timeout(self): + deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + self.assertRaises(errors.TimeoutError, self.client.finalize_order, self.orderr, deadline) + + class MockJSONDeSerializable(jose.JSONDeSerializable): # pylint: disable=missing-docstring def __init__(self, value): diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 624ccb3d9..991335958 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -89,7 +89,7 @@ class ValidationError(Error): """ def __init__(self, failed_authzrs): self.failed_authzrs = failed_authzrs - super(ClientError, self).__init__() + super(ValidationError, self).__init__() class TimeoutError(Error): """Error for when polling an authorization or an order times out.""" @@ -103,6 +103,7 @@ class IssuanceError(Error): :param messages.Error error: The error provided by the server. """ self.error = error + super(IssuanceError, self).__init__() class ConflictError(ClientError): """Error for when the server returns a 409 (Conflict) HTTP status. diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 418bcead9..23cd66c63 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -509,9 +509,6 @@ class Revocation(jose.JSONObjectWithFields): class Order(ResourceBody): """Order Resource Body. - .. note:: Parsing of identifiers on response doesn't work right now; to make - it work we would need to set up the equivalent of Identifier.from_json, but - for a list. :ivar list of .Identifier: List of identifiers for the certificate. :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. @@ -530,6 +527,10 @@ class Order(ResourceBody): expires = fields.RFC3339Field('expires', omitempty=True) error = jose.Field('error', omitempty=True, decoder=Error.from_json) + @identifiers.decoder + def identifiers(value): # pylint: disable=missing-docstring,no-self-argument + return tuple(Identifier.from_json(identifier) for identifier in value) + class OrderResource(ResourceWithURI): """Order Resource. From adec7a8feda1b6ad4f989d4c293fdb78a646917a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Feb 2018 09:51:27 -0800 Subject: [PATCH 35/93] Cleanup dockerfile-dev (#5435) * cleanup dockerfile-dev * map port 80 * remove python3-dev package --- Dockerfile-dev | 65 ++++++---------------------------------------- docker-compose.yml | 1 + 2 files changed, 9 insertions(+), 57 deletions(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 581b58f11..9e35ebec8 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,70 +1,21 @@ # This Dockerfile builds an image for development. -FROM ubuntu:trusty -MAINTAINER Jakub Warmuz -MAINTAINER William Budington -MAINTAINER Yan +FROM ubuntu:xenial -# Note: this only exposes the port to other docker containers. You -# still have to bind to 443@host at runtime, as per the ACME spec. -EXPOSE 443 - -# TODO: make sure --config-dir and --work-dir cannot be changed -# through the CLI (certbot-docker wrapper that uses standalone -# authenticator and text mode only?) -VOLUME /etc/letsencrypt /var/lib/letsencrypt +# Note: this only exposes the port to other docker containers. +EXPOSE 80 443 WORKDIR /opt/certbot/src -# no need to mkdir anything: -# https://docs.docker.com/reference/builder/#copy -# If doesn't exist, it is created along with all missing -# directories in its path. - # TODO: Install Apache/Nginx for plugin development. -COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto -RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ - apt-get install python3-dev git -y && \ +COPY . . +RUN apt-get update && \ + apt-get install apache2 git nginx-light -y && \ + letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ /var/tmp/* -# the above is not likely to change, so by putting it further up the -# Dockerfile we make sure we cache as much as possible - -COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ - -# all above files are necessary for setup.py, however, package source -# code directory has to be copied separately to a subdirectory... -# https://docs.docker.com/reference/builder/#copy: "If is a -# directory, the entire contents of the directory are copied, -# including filesystem metadata. Note: The directory itself is not -# copied, just its contents." Order again matters, three files are far -# more likely to be cached than the whole project directory - -COPY certbot /opt/certbot/src/certbot/ -COPY acme /opt/certbot/src/acme/ -COPY certbot-apache /opt/certbot/src/certbot-apache/ -COPY certbot-nginx /opt/certbot/src/certbot-nginx/ -COPY letshelp-certbot /opt/certbot/src/letshelp-certbot/ -COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ -COPY tests /opt/certbot/src/tests/ - -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ - /opt/certbot/venv/bin/pip install -U pip && \ - /opt/certbot/venv/bin/pip install -U setuptools && \ - /opt/certbot/venv/bin/pip install \ - -e /opt/certbot/src/acme \ - -e /opt/certbot/src \ - -e /opt/certbot/src/certbot-apache \ - -e /opt/certbot/src/certbot-nginx \ - -e /opt/certbot/src/letshelp-certbot \ - -e /opt/certbot/src/certbot-compatibility-test \ - -e /opt/certbot/src[dev,docs] - -# install in editable mode (-e) to save space: it's not possible to -# "rm -rf /opt/certbot/src" (it's stays in the underlaying image); -# this might also help in debugging: you can "docker run --entrypoint -# bash" and investigate, apply patches, etc. +RUN VENV_NAME="../venv" tools/venv.sh ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/docker-compose.yml b/docker-compose.yml index 00d3d4c72..75a5b9aab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ services: context: . dockerfile: Dockerfile-dev ports: + - "80:80" - "443:443" volumes: - .:/opt/certbot/src From 2a142aa93288d0db87b8f12dc71fce70ee3ce482 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Feb 2018 14:47:10 -0800 Subject: [PATCH 36/93] Make Certbot depend on josepy (#5542) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ce505a62e..47b5b0b2c 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ install_requires = [ 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=1.2', # load_pem_x509_certificate + 'josepy', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', From e95e963ad62735a0cd23a46e27a7cbef3790afb2 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 16 Feb 2018 16:05:16 -0800 Subject: [PATCH 37/93] Get common name from CSR in new_order in ClientV2 (#5587) * switch new_order to use crypto_util._pyopenssl_cert_or_req_san * move certbot.crypto_util._get_names_from_loaded_cert_or_req functionality to acme.crypto_util._pyopenssl_cert_or_req_all_names --- acme/acme/client.py | 10 ++++------ acme/acme/crypto_util.py | 9 +++++++++ acme/acme/crypto_util_test.py | 24 ++++++++++++++++++++++++ acme/acme/testdata/cert-nocn.der | Bin 0 -> 1397 bytes certbot/crypto_util.py | 8 +------- 5 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 acme/acme/testdata/cert-nocn.der diff --git a/acme/acme/client.py b/acme/acme/client.py index bc93ca06f..1f4ae4fad 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,7 +1,6 @@ """ACME client API.""" import base64 import collections -import cryptography import datetime from email.utils import parsedate_tz import heapq @@ -17,6 +16,7 @@ import re import requests import sys +from acme import crypto_util from acme import errors from acme import jws from acme import messages @@ -568,11 +568,9 @@ class ClientV2(ClientBase): :returns: The newly created order. :rtype: OrderResource """ - csr = cryptography.x509.load_pem_x509_csr(csr_pem, - cryptography.hazmat.backends.default_backend()) - san_extension = next(ext for ext in csr.extensions - if ext.oid == cryptography.x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) - dnsNames = san_extension.value.get_values_for_type(cryptography.x509.DNSName) + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) identifiers = [] for name in dnsNames: diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index b8fba0348..a986721f0 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -186,6 +186,15 @@ def make_csr(private_key_pem, domains, must_staple=False): return OpenSSL.crypto.dump_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr) +def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): + common_name = loaded_cert_or_req.get_subject().CN + sans = _pyopenssl_cert_or_req_san(loaded_cert_or_req) + + if common_name is None: + return sans + else: + return [common_name] + [d for d in sans if d != common_name] + def _pyopenssl_cert_or_req_san(cert_or_req): """Get Subject Alternative Names from certificate or CSR using pyOpenSSL. diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf..14aaac8b5 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -65,6 +65,30 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # self.assertRaises(errors.Error, self._probe, b'bar') +class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): + """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" + + @classmethod + def _call(cls, loader, name): + # pylint: disable=protected-access + from acme.crypto_util import _pyopenssl_cert_or_req_all_names + return _pyopenssl_cert_or_req_all_names(loader(name)) + + def _call_cert(self, name): + return self._call(test_util.load_cert, name) + + def test_cert_one_san_no_common(self): + self.assertEqual(self._call_cert('cert-nocn.der'), + ['no-common-name.badssl.com']) + + def test_cert_no_sans_yes_common(self): + self.assertEqual(self._call_cert('cert.pem'), ['example.com']) + + def test_cert_two_sans_yes_common(self): + self.assertEqual(self._call_cert('cert-san.pem'), + ['example.com', 'www.example.com']) + + class PyOpenSSLCertOrReqSANTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_san.""" diff --git a/acme/acme/testdata/cert-nocn.der b/acme/acme/testdata/cert-nocn.der new file mode 100644 index 0000000000000000000000000000000000000000..59da83ccc6dc3f2a05fe762d6d3372cb446d88ac GIT binary patch literal 1397 zcmXqLVl6aiVu@V9%*4pVB#@u9Us#IesM(YUabM(~3ac9MvT?DFX?R2$!&+v%jyui@$=iqk>OnZe~epilL2x6-bU**cd7o6zr(rUzDDhmsyoq zl9`{U5SEyenF3)3rzV#cr78rc7L@_*baph56X!KFH!wCbHUNVtab6<>12ZEdBV$ub zQ_CoWY$SJw1{(?+2twS=Ed4`e(;vm)B!UDdfIVG98F8R5MnRyDq!9E5}j7rD>$H>aS+{DPw02Jq9 zYGPz$IK=YiN6$N!Lz`~|TrG$=Gg;zw%!&-p4N6;oPMx<=Tc^w}=YV&DkI$=3TMqCy zY}@d&=wuty<y8VeR_DBi?&nPSFc6UzRSD*z?WN>8RUfv3B+OC9k>W3I?eu7U#71 zRE9penEisy)$G-j&A)YDMK2k9 zJXb0_en_M$N8^>b*Zo&HTZ^{TMJ*D$lesb6U)Rz-AiH|l^)LeDLwhwS|v73 zy^Xs(Wq8!*E&kp8-JytK%_NSSJ>f0EjjX4i{mTxS;h^4nTwr%`!TU@JM|RiHnx855 zwmOGb*1Vo9<z44-7_GWflnou?CS@pS5qDU+ce3f47#0 zhnCQGX{UoK2C^Upd@N!tB6HsztZwACa?dQ3s1-Dyyl1Jj{cmt0ljUb*{LjL|%*49D zfCr>p7{q5XV1Q^=Wf3zFVdKzdV`ODzXJ&-6m<$3yiWOKq4crZ^*?52oSQs}MGBPnT zvlwU_Xu=dSF^b7%l#~<{Tj}c;gOi?Ka(-@pN?+{ z-k{bX6vj7TLb9iXGJ8r4RN%%kwn;#ppPyV@fMPCC70{=GEV2gDO_F)}y1<~w&Ck=# zOUzBxOG->BE(UuSIU@i|5MV}NWXSxZ>ULmW+UMOr#Oj|(nYf(rnDb6zkyW|Xz_ z$9{(K%Jus_l6IO3g=HA){+awKO)O|fmTT1Z+#7Ryp6LWmIpcR=gMC-d`J0n;Teo@| z6zW||Pht60z4=n>rj4&JFaGB1)RMV*x5}HhJC+@K>piVA&uZReo2JynpBFM}&U!55 z^j+hWs<_`F(Vy8@W!9tBm(mqy|NpbeP?kwtdY#U4)jc=2tozb#`ZwppYdhvmr}>U7 l*UbLhr)aUs$L-SL{YJ0C_h)*Gh(7yrDof|BxSMr#8URDu=Gy=O literal 0 HcmV?d00001 diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 3ae16529d..8368855cd 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -340,14 +340,8 @@ def _get_names_from_cert_or_req(cert_or_req, load_func, typ): def _get_names_from_loaded_cert_or_req(loaded_cert_or_req): - common_name = loaded_cert_or_req.get_subject().CN # pylint: disable=protected-access - sans = acme_crypto_util._pyopenssl_cert_or_req_san(loaded_cert_or_req) - - if common_name is None: - return sans - else: - return [common_name] + [d for d in sans if d != common_name] + return acme_crypto_util._pyopenssl_cert_or_req_all_names(loaded_cert_or_req) def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): From 42638afc7568c161f467a4f6f385d597a6e329d9 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 26 Jan 2018 11:47:25 +0200 Subject: [PATCH 38/93] Drop support for EOL Python 2.6 and 3.3 * Drop support for EOL Python 2.6 * Use more helpful assertIn/NotIn instead of assertTrue/False * Drop support for EOL Python 3.3 * Remove redundant Python 3.3 code * Restore code for RHEL 6 and virtualenv for Py2.7 * Revert pipstrap.py to upstream * Merge py26_packages and non_py26_packages into all_packages * Revert changes to *-auto in root * Update by calling letsencrypt-auto-source/build.py * Revert permissions for pipstrap.py --- .travis.yml | 8 ---- acme/acme/__init__.py | 10 ---- acme/acme/crypto_util.py | 4 +- acme/acme/crypto_util_test.py | 12 ++--- acme/setup.py | 9 ---- certbot-apache/setup.py | 2 - certbot-compatibility-test/setup.py | 2 - certbot-dns-cloudflare/setup.py | 2 - certbot-dns-cloudxns/setup.py | 1 - certbot-dns-digitalocean/setup.py | 2 - certbot-dns-dnsimple/setup.py | 1 - certbot-dns-dnsmadeeasy/setup.py | 1 - certbot-dns-google/setup.py | 2 - certbot-dns-luadns/setup.py | 1 - certbot-dns-nsone/setup.py | 1 - certbot-dns-rfc2136/setup.py | 2 - certbot-dns-route53/setup.py | 2 - certbot-nginx/setup.py | 2 - certbot/log.py | 48 ++++--------------- certbot/main.py | 12 ----- docs/contributing.rst | 2 +- docs/install.rst | 2 +- letsencrypt-auto-source/letsencrypt-auto | 35 ++++---------- .../letsencrypt-auto.template | 35 ++++---------- letsencrypt-auto-source/tests/auto_test.py | 16 +++---- letshelp-certbot/letshelp_certbot/apache.py | 4 +- letshelp-certbot/setup.py | 2 - setup.py | 9 ---- tests/letstest/scripts/test_tox.sh | 8 +--- tests/run_http_server.py | 2 +- tools/install_and_test.sh | 4 -- tox.ini | 41 ++++------------ 32 files changed, 56 insertions(+), 228 deletions(-) diff --git a/.travis.yml b/.travis.yml index 35666d8e6..1077d99d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,18 +25,10 @@ matrix: addons: - python: "2.7" env: TOXENV=lint - - python: "2.6" - env: TOXENV=py26 - sudo: required - services: docker - python: "2.7" env: TOXENV=py27-oldest sudo: required services: docker - - python: "3.3" - env: TOXENV=py33 - sudo: required - services: docker - python: "3.6" env: TOXENV=py36 sudo: required diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index 5850fa955..e8a0b16a8 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -10,13 +10,3 @@ supported version: `draft-ietf-acme-01`_. https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ -import sys -import warnings - -for (major, minor) in [(2, 6), (3, 3)]: - if sys.version_info[:2] == (major, minor): - warnings.warn( - "Python {0}.{1} support will be dropped in the next release of " - "acme. Please upgrade your Python version.".format(major, minor), - DeprecationWarning, - ) #pragma: no cover diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index b8fba0348..78ba41d0f 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -5,7 +5,6 @@ import logging import os import re import socket -import sys import OpenSSL @@ -130,8 +129,7 @@ def probe_sni(name, host, port=443, timeout=300, context = OpenSSL.SSL.Context(method) context.set_timeout(timeout) - socket_kwargs = {} if sys.version_info < (2, 7) else { - 'source_address': source_address} + socket_kwargs = {'source_address': source_address} host_protocol_agnostic = None if host == '::' or host == '0' else host diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf..22a507811 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -170,9 +170,9 @@ class MakeCSRTest(unittest.TestCase): self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 1) self.assertEquals(csr.get_extensions()[0].get_data(), @@ -188,9 +188,9 @@ class MakeCSRTest(unittest.TestCase): csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 2) # NOTE: Ideally we would filter by the TLS Feature OID, but diff --git a/acme/setup.py b/acme/setup.py index ce426cf74..ba5c8e6fb 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -25,13 +25,6 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ 'pytest', 'pytest-xdist', @@ -58,10 +51,8 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 38f41e9f1..d7c223a0a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 8f9f897cf..7e1b059e2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -40,10 +40,8 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 612e7259f..d619f1872 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -39,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3157400c6..5d14f3e29 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1a68400fa..ce8fedd46 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 35de47308..06af16759 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a946d00a4..7c0f3ed86 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8585fc848..de881ad84 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -44,10 +44,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4fec37e29..0d580b7ee 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index dca9ebf27..c0ba11470 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bfa72b50b..5161e7a94 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -39,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8df687972..09f8a7d52 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -32,10 +32,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 152f77de8..37c477ef6 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot/log.py b/certbot/log.py index f7c7b126c..e0d2e8f11 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -165,12 +165,7 @@ class ColoredStreamHandler(logging.StreamHandler): """ def __init__(self, stream=None): - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(ColoredStreamHandler, self).__init__(stream) + super(ColoredStreamHandler, self).__init__(stream) self.colored = (sys.stderr.isatty() if stream is None else stream.isatty()) self.red_level = logging.WARNING @@ -184,9 +179,7 @@ class ColoredStreamHandler(logging.StreamHandler): :rtype: str """ - out = (logging.StreamHandler.format(self, record) - if sys.version_info < (2, 7) - else super(ColoredStreamHandler, self).format(record)) + out = super(ColoredStreamHandler, self).format(record) if self.colored and record.levelno >= self.red_level: return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) else: @@ -203,23 +196,14 @@ class MemoryHandler(logging.handlers.MemoryHandler): def __init__(self, target=None): # capacity doesn't matter because should_flush() is overridden capacity = float('inf') - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.__init__( - self, capacity, target=target) - else: - super(MemoryHandler, self).__init__(capacity, target=target) + super(MemoryHandler, self).__init__(capacity, target=target) def close(self): """Close the memory handler, but don't set the target to None.""" # This allows the logging module which may only have a weak # reference to the target handler to properly flush and close it. target = self.target - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.close(self) - else: - super(MemoryHandler, self).close() + super(MemoryHandler, self).close() self.target = target def flush(self, force=False): # pylint: disable=arguments-differ @@ -233,10 +217,7 @@ class MemoryHandler(logging.handlers.MemoryHandler): # This method allows flush() calls in logging.shutdown to be a # noop so we can control when this handler is flushed. if force: - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.flush(self) - else: - super(MemoryHandler, self).flush() + super(MemoryHandler, self).flush() def shouldFlush(self, record): """Should the buffer be automatically flushed? @@ -262,12 +243,7 @@ class TempHandler(logging.StreamHandler): """ def __init__(self): stream = tempfile.NamedTemporaryFile('w', delete=False) - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(TempHandler, self).__init__(stream) + super(TempHandler, self).__init__(stream) self.path = stream.name self._delete = True @@ -278,12 +254,7 @@ class TempHandler(logging.StreamHandler): """ self._delete = False - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.emit(self, record) - else: - super(TempHandler, self).emit(record) + super(TempHandler, self).emit(record) def close(self): """Close the handler and the temporary log file. @@ -299,10 +270,7 @@ class TempHandler(logging.StreamHandler): if self._delete: os.remove(self.path) self._delete = False - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.close(self) - else: - super(TempHandler, self).close() + super(TempHandler, self).close() finally: self.release() diff --git a/certbot/main.py b/certbot/main.py index 32dd69256..76a90d499 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -4,7 +4,6 @@ import functools import logging.handlers import os import sys -import warnings import configobj import josepy as jose @@ -1218,17 +1217,6 @@ def main(cli_args=sys.argv[1:]): # Let plugins_cmd be run as un-privileged user. if config.func != plugins_cmd: raise - deprecation_fmt = ( - "Python %s.%s support will be dropped in the next " - "release of Certbot - please upgrade your Python version.") - # We use the warnings system for Python 2.6 and logging for Python 3 - # because DeprecationWarnings are only reported by default in Python <= 2.6 - # and warnings can be disabled by the user. - if sys.version_info[:2] == (2, 6): - warning = deprecation_fmt % sys.version_info[:2] - warnings.warn(warning, DeprecationWarning) - elif sys.version_info[:2] == (3, 3): - logger.warning(deprecation_fmt, *sys.version_info[:2]) set_displayer(config) diff --git a/docs/contributing.rst b/docs/contributing.rst index 83b607e15..654528e3d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -422,7 +422,7 @@ OS-level dependencies can be installed like so: In general... * ``sudo`` is required as a suggested way of running privileged process -* `Python`_ 2.6/2.7 is required +* `Python`_ 2.7 is required * `Augeas`_ is required for the Python bindings * ``virtualenv`` and ``pip`` are used for managing other python library dependencies diff --git a/docs/install.rst b/docs/install.rst index c18c3cdbc..aec885b62 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,7 +22,7 @@ your system. System Requirements =================== -Certbot currently requires Python 2.6, 2.7, or 3.3+. By default, it requires +Certbot currently requires Python 2.7, or 3.4+. By default, it requires root access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and modify webserver diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8ff7944b5..aed15a8ef 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -246,7 +246,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -781,20 +781,11 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -965,18 +956,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2ce337002..b3d6ab740 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -246,7 +246,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -320,20 +320,11 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -504,18 +495,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index d187452a1..8c2bfc079 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -287,8 +287,8 @@ class AutoTests(TestCase): self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: - self.assertTrue('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This # conveniently sets us up to test the next 2 cases. @@ -296,8 +296,8 @@ class AutoTests(TestCase): # Test when neither phase-1 upgrade nor phase-2 upgrade is # needed (probably a common case): out, err = run_letsencrypt_auto() - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertFalse('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertNotIn('Creating virtual environment...', out) def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" @@ -312,8 +312,8 @@ class AutoTests(TestCase): # Create venv saving the correct bootstrap script version out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: bootstrap_version = f.read() @@ -329,8 +329,8 @@ class AutoTests(TestCase): out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) + self.assertNotIn('Upgrading certbot-auto ', out) + self.assertIn('Creating virtual environment...', out) def test_openssl_failure(self): """Make sure we stop if the openssl signature check fails.""" diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index b13057ca5..f77a6a1b0 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -5,7 +5,6 @@ from __future__ import print_function import argparse import atexit -import contextlib import os import re import shutil @@ -302,8 +301,7 @@ def main(): make_and_verify_selection(args.server_root, tempdir) tarpath = os.path.join(tempdir, "config.tar.gz") - # contextlib.closing used for py26 support - with contextlib.closing(tarfile.open(tarpath, mode="w:gz")) as tar: + with tarfile.open(tarpath, mode="w:gz") as tar: tar.add(tempdir, arcname=".") # TODO: Submit tarpath diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 3ce442b3e..7c8c39068 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -31,10 +31,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/setup.py b/setup.py index ce505a62e..e3824a7f7 100644 --- a/setup.py +++ b/setup.py @@ -52,13 +52,6 @@ install_requires = [ 'zope.interface', ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', @@ -98,10 +91,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index 4c2eb429e..84e4bcd22 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -15,10 +15,4 @@ VENV_BIN=${VENV_PATH}/bin cd letsencrypt ./tools/venv.sh -PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - -if [ $PYVER -eq 26 ] ; then - venv/bin/tox -e py26 -else - venv/bin/tox -e py27 -fi +venv/bin/tox -e py27 diff --git a/tests/run_http_server.py b/tests/run_http_server.py index fd1163816..0e4f8ac79 100644 --- a/tests/run_http_server.py +++ b/tests/run_http_server.py @@ -3,7 +3,7 @@ import sys # Run Python's built-in HTTP server # Usage: python ./tests/run_http_server.py port_num -# NOTE: This script should be compatible with 2.6, 2.7, 3.3+ +# NOTE: This script should be compatible with 2.7, 3.4+ # sys.argv (port number) is passed as-is to the HTTP server module runpy.run_module( diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 0d39e0594..f0385470b 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -18,10 +18,6 @@ for requirement in "$@" ; do pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] if [ $pkg = "." ]; then pkg="certbot" - else - # Work around a bug in pytest/importlib for the deprecated Python 3.3. - # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. - pkg=$(echo "$pkg" | tr - _) fi "$(dirname $0)/pytest.sh" --pyargs $pkg done diff --git a/tox.ini b/tox.ini index 20f5cda32..971aa7631 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py{26,33,34,35,36},cover,lint +envlist = modification,py{34,35,36},cover,lint [base] # pip installs the requested packages in editable mode @@ -14,25 +14,22 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -py26_packages = +all_packages = acme[dev] \ .[dev] \ certbot-apache \ certbot-dns-cloudflare \ + certbot-dns-cloudxns \ certbot-dns-digitalocean \ + certbot-dns-dnsimple \ + certbot-dns-dnsmadeeasy \ certbot-dns-google \ + certbot-dns-luadns \ + certbot-dns-nsone \ certbot-dns-rfc2136 \ certbot-dns-route53 \ certbot-nginx \ letshelp-certbot -non_py26_packages = - certbot-dns-cloudxns \ - certbot-dns-dnsimple \ - certbot-dns-dnsmadeeasy \ - certbot-dns-luadns \ - certbot-dns-nsone -all_packages = - {[base]py26_packages} {[base]non_py26_packages} install_packages = {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} source_paths = @@ -54,32 +51,15 @@ source_paths = letshelp-certbot/letshelp_certbot tests/lock_test.py -[testenv:py26] -commands = - {[base]install_and_test} {[base]py26_packages} - python tests/lock_test.py -deps = - setuptools==36.8.0 - wheel==0.29.0 -passenv = TRAVIS - [testenv] commands = - {[testenv:py26]commands} - {[base]install_and_test} {[base]non_py26_packages} + {[base]install_and_test} {[base]all_packages} + python tests/lock_test.py setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 passenv = - {[testenv:py26]passenv} - -[testenv:py33] -commands = - {[testenv]commands} -deps = - wheel==0.29.0 -passenv = - {[testenv]passenv} + TRAVIS [testenv:py27-oldest] commands = @@ -104,7 +84,6 @@ passenv = {[testenv]passenv} [testenv:lint] -# recent versions of pylint do not support Python 2.6 (#97, #187) basepython = python2.7 # separating into multiple invocations disables cross package # duplicate code checking; if one of the commands fails, others will From 73bd801f352fb9bad7fe8bc35c368f573c15a21e Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:21:02 -0800 Subject: [PATCH 39/93] add and use request_authorizations --- acme/acme/client.py | 14 ++++++++++++++ certbot/auth_handler.py | 10 ++++++---- certbot/client.py | 23 ++++++++--------------- certbot/main.py | 2 +- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 1f4ae4fad..6b4d65233 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -671,6 +671,7 @@ class BackwardsCompatibleClientV2(object): self.client = Client(directory, key=key, net=net) else: self.client = ClientV2(directory, net=net) + self.orderr = None def __getattr__(self, name): if name in vars(self.client): @@ -705,6 +706,19 @@ class BackwardsCompatibleClientV2(object): regr = regr.update(terms_of_service_agreed=True) return self.client.new_account(regr) + def request_authorizations(self, csr_pem): + if self.acme_version == 1: + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) + authorizations = [] + for domain in dnsNames: + authorizations.append(self.client.request_domain_challenges(domain)) + return authorizations + else: + self.orderr = self.client.new_order(csr_pem) + return self.orderr.authorizations + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 5f520cbcb..4f88199e3 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,10 +48,10 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def get_authorizations(self, domains, best_effort=False): + def get_authorizations(self, csr_pem, best_effort=False): """Retrieve all authorizations for challenges. - :param list domains: Domains for authorization + :param list csr_pem: CSR containing domains for authorization :param bool best_effort: Whether or not all authorizations are required (this is useful in renewal) @@ -62,8 +62,10 @@ class AuthHandler(object): authorizations """ - for domain in domains: - self.authzr[domain] = self.acme.request_domain_challenges(domain) + authzrs = self.acme.request_authorizations(csr_pem) + for authzr in authzrs: + self.authzr[authzr.body.identifier.value] = authzr + domains = self.authzr.keys() self._choose_challenges(domains) config = zope.component.getUtility(interfaces.IConfig) diff --git a/certbot/client.py b/certbot/client.py index 67ee8f7fa..8e3ec6c62 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,13 +235,9 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, domains, csr, authzr=None): + def obtain_certificate_from_csr(self, csr, authzr=None): """Obtain certificate. - Internal function with precondition that `domains` are - consistent with identifiers present in the `csr`. - - :param list domains: Domain names. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. @@ -261,10 +257,10 @@ class Client(object): if self.account.regr is None: raise errors.Error("Please register with the ACME server first.") - logger.debug("CSR: %s, domains: %s", csr, domains) + logger.debug("CSR: %s", csr) if authzr is None: - authzr = self.auth_handler.get_authorizations(domains) + authzr = self.auth_handler.get_authorizations(csr) certr = self.acme.request_issuance( jose.ComparableX509( @@ -307,13 +303,6 @@ class Client(object): :rtype: tuple """ - authzr = self.auth_handler.get_authorizations( - domains, - self.config.allow_subset_of_names) - - auth_domains = set(a.body.identifier.value for a in authzr) - domains = [d for d in domains if d in auth_domains] - # Create CSR from names if self.config.dry_run: key = util.Key(file=None, @@ -326,8 +315,12 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) + authzr = self.auth_handler.get_authorizations( + csr, + self.config.allow_subset_of_names) + certr, chain = self.obtain_certificate_from_csr( - domains, csr, authzr=authzr) + csr, authzr=authzr) return certr, chain, key, csr diff --git a/certbot/main.py b/certbot/main.py index ff3758985..d01f68920 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1064,7 +1064,7 @@ def _csr_get_and_save_cert(config, le_client): """ csr, _ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr) + certr, chain = le_client.obtain_certificate_from_csr(csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) From eaf739184cf517d5a3f5103caa072bb0cd39b4e9 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:29:42 -0800 Subject: [PATCH 40/93] pass pem to auth_handler --- certbot/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 8e3ec6c62..d7d2acb14 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -260,7 +260,7 @@ class Client(object): logger.debug("CSR: %s", csr) if authzr is None: - authzr = self.auth_handler.get_authorizations(csr) + authzr = self.auth_handler.get_authorizations(csr.data) certr = self.acme.request_issuance( jose.ComparableX509( @@ -316,7 +316,7 @@ class Client(object): csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) authzr = self.auth_handler.get_authorizations( - csr, + csr.data, self.config.allow_subset_of_names) certr, chain = self.obtain_certificate_from_csr( From ea2022588b4d95f1d14849b918b2cd3788cc6084 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 16:32:49 -0800 Subject: [PATCH 41/93] add docstring --- acme/acme/client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/acme/acme/client.py b/acme/acme/client.py index 6b4d65233..e7cf016bb 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -707,6 +707,16 @@ class BackwardsCompatibleClientV2(object): return self.client.new_account(regr) def request_authorizations(self, csr_pem): + """Request authorizations for the domains in csr_pem. + + Calls request_domain_challenges for each domain for V1, and + calls new_order and saves the result for V2. + + :param str csr_pem: A CSR in PEM format. + + :returns: List of Authorization Resources. + :rtype: list of `.AuthorizationResource` + """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) # pylint: disable=protected-access From 20d0b91c710bd8110aa5ee23082f45583e7789a4 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 17:35:10 -0800 Subject: [PATCH 42/93] switch interface to new_order and remove best_effort flag --- acme/acme/client.py | 18 ++++++++---------- certbot/auth_handler.py | 27 ++++++++++----------------- certbot/client.py | 15 ++++----------- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index e7cf016bb..1838fab42 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -671,7 +671,6 @@ class BackwardsCompatibleClientV2(object): self.client = Client(directory, key=key, net=net) else: self.client = ClientV2(directory, net=net) - self.orderr = None def __getattr__(self, name): if name in vars(self.client): @@ -706,16 +705,16 @@ class BackwardsCompatibleClientV2(object): regr = regr.update(terms_of_service_agreed=True) return self.client.new_account(regr) - def request_authorizations(self, csr_pem): - """Request authorizations for the domains in csr_pem. + def new_order(self, csr_pem): + """Request a new Order object from the server. - Calls request_domain_challenges for each domain for V1, and - calls new_order and saves the result for V2. + If using ACMEv1, returns a dummy OrderResource with only + the authorizations field filled in. :param str csr_pem: A CSR in PEM format. - :returns: List of Authorization Resources. - :rtype: list of `.AuthorizationResource` + :returns: The newly created order. + :rtype: OrderResource """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) @@ -724,10 +723,9 @@ class BackwardsCompatibleClientV2(object): authorizations = [] for domain in dnsNames: authorizations.append(self.client.request_domain_challenges(domain)) - return authorizations + return messages.OrderResource(authorizations=authorizations) else: - self.orderr = self.client.new_order(csr_pem) - return self.orderr.authorizations + return self.client.new_order(csr_pem) def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 4f88199e3..825513329 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,12 +48,11 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def get_authorizations(self, csr_pem, best_effort=False): + def handle_authorizations(self, orderr): """Retrieve all authorizations for challenges. - :param list csr_pem: CSR containing domains for authorization - :param bool best_effort: Whether or not all authorizations are - required (this is useful in renewal) + :param acme.messages.OrderResource orderr: must have + authorizations filled in :returns: List of authorization resources :rtype: list @@ -62,7 +61,7 @@ class AuthHandler(object): authorizations """ - authzrs = self.acme.request_authorizations(csr_pem) + authzrs = orderr.authorizations for authzr in authzrs: self.authzr[authzr.body.identifier.value] = authzr domains = self.authzr.keys() @@ -80,7 +79,7 @@ class AuthHandler(object): 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp, best_effort) + self._respond(resp) # Just make sure all decisions are complete. self.verify_authzr_complete() @@ -124,7 +123,7 @@ class AuthHandler(object): return resp - def _respond(self, resp, best_effort): + def _respond(self, resp): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -137,7 +136,7 @@ class AuthHandler(object): # Check for updated status... try: - self._poll_challenges(chall_update, best_effort) + self._poll_challenges(chall_update) finally: # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) @@ -169,7 +168,7 @@ class AuthHandler(object): return active_achalls def _poll_challenges( - self, chall_update, best_effort, min_sleep=3, max_rounds=15): + self, chall_update, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" dom_to_check = set(chall_update.keys()) comp_domains = set() @@ -190,14 +189,8 @@ class AuthHandler(object): chall_update[domain].remove(achall) # We failed some challenges... damage control else: - if best_effort: - comp_domains.add(domain) - logger.warning( - "Challenge failed for domain %s", - domain) - else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) + all_failed_achalls.update( + updated for _, updated in failed_achalls) if all_failed_achalls: _report_failed_challs(all_failed_achalls) diff --git a/certbot/client.py b/certbot/client.py index d7d2acb14..61e9db635 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,14 +235,12 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr, authzr=None): + def obtain_certificate_from_csr(self, csr): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. - :param list authzr: List of - :class:`acme.messages.AuthorizationResource` :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -259,8 +257,8 @@ class Client(object): logger.debug("CSR: %s", csr) - if authzr is None: - authzr = self.auth_handler.get_authorizations(csr.data) + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr) certr = self.acme.request_issuance( jose.ComparableX509( @@ -315,12 +313,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - authzr = self.auth_handler.get_authorizations( - csr.data, - self.config.allow_subset_of_names) - - certr, chain = self.obtain_certificate_from_csr( - csr, authzr=authzr) + certr, chain = self.obtain_certificate_from_csr(csr) return certr, chain, key, csr From 68e24a8ea7eeb405592e40ed340baff6f3f3821b Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 16 Feb 2018 17:59:51 -0800 Subject: [PATCH 43/93] start test updates --- certbot/tests/auth_handler_test.py | 54 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 32c4c0d3b..d424e59ca 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -57,8 +57,8 @@ class ChallengeFactoryTest(unittest.TestCase): errors.Error, self.handler._challenge_factory, "failure.com", [0]) -class GetAuthorizationsTest(unittest.TestCase): - """get_authorizations test. +class HandleAuthorizationsTest(unittest.TestCase): + """handle_authorizations test. This tests everything except for all functions under _poll_challenges. @@ -92,12 +92,11 @@ class GetAuthorizationsTest(unittest.TestCase): @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_name1_tls_sni_01_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) - mock_poll.side_effect = self._validate_all - authzr = self.handler.get_authorizations(["0"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) @@ -115,14 +114,13 @@ class GetAuthorizationsTest(unittest.TestCase): @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES, combos=False) - mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = self.handler.get_authorizations(["0"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -146,7 +144,11 @@ class GetAuthorizationsTest(unittest.TestCase): mock_poll.side_effect = self._validate_all - authzr = self.handler.get_authorizations(["0", "1", "2"]) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -169,31 +171,33 @@ class GetAuthorizationsTest(unittest.TestCase): def test_debug_challenges(self, mock_poll): zope.component.provideUtility( mock.Mock(debug_challenges=True), interfaces.IConfig) - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) mock_poll.side_effect = self._validate_all - self.handler.get_authorizations(["0"]) + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(self.mock_display.notification.call_count, 1) def test_perform_failure(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + self.mock_auth.perform.side_effect = errors.AuthorizationError self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) def test_no_domains(self): - self.assertRaises(errors.AuthorizationError, self.handler.get_authorizations, []) + mock_order = mock.MagicMock(authorizations=[]) + self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_preferred_challenge_choice(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) @@ -201,20 +205,20 @@ class GetAuthorizationsTest(unittest.TestCase): self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - self.handler.get_authorizations(["0"]) + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") def test_preferred_challenges_not_supported(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) self.handler.pref_challs.append(challenges.HTTP01.typ) self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1, unused_2): + def _validate_all(self, unused_1): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] self.handler.authzr[dom] = acme_util.gen_authzr( From 9c84fe1144cc5ccec5fc9cfcdf7c5adf8cfdf15c Mon Sep 17 00:00:00 2001 From: Matt Christian Date: Sun, 18 Feb 2018 15:45:22 -0600 Subject: [PATCH 44/93] Add override class for ID="ol" AKA Oracle Linux Server, a clone of CentOS/RHEL. --- certbot-apache/certbot_apache/entrypoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-apache/certbot_apache/entrypoint.py b/certbot-apache/certbot_apache/entrypoint.py index 4267398d5..6f1443507 100644 --- a/certbot-apache/certbot_apache/entrypoint.py +++ b/certbot-apache/certbot_apache/entrypoint.py @@ -17,6 +17,7 @@ OVERRIDE_CLASSES = { "centos": override_centos.CentOSConfigurator, "centos linux": override_centos.CentOSConfigurator, "fedora": override_centos.CentOSConfigurator, + "ol": override_centos.CentOSConfigurator, "red hat enterprise linux server": override_centos.CentOSConfigurator, "rhel": override_centos.CentOSConfigurator, "amazon": override_centos.CentOSConfigurator, From d6b4e2001b404b1f9bd4c3929e888726d583ee57 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 13:19:04 -0800 Subject: [PATCH 45/93] put back in best_effort code, with a todo for actually supporting it in ACMEv2 --- certbot/auth_handler.py | 22 +++++++++++++++------- certbot/client.py | 11 ++++++++--- certbot/tests/auth_handler_test.py | 2 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 825513329..662cadc65 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -48,11 +48,13 @@ class AuthHandler(object): # List must be used to keep responses straight. self.achalls = [] - def handle_authorizations(self, orderr): + def handle_authorizations(self, orderr, best_effort=False): """Retrieve all authorizations for challenges. :param acme.messages.OrderResource orderr: must have authorizations filled in + :param bool best_effort: Whether or not all authorizations are + required (this is useful in renewal) :returns: List of authorization resources :rtype: list @@ -79,7 +81,7 @@ class AuthHandler(object): 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp) + self._respond(resp, best_effort) # Just make sure all decisions are complete. self.verify_authzr_complete() @@ -123,7 +125,7 @@ class AuthHandler(object): return resp - def _respond(self, resp): + def _respond(self, resp, best_effort): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -136,7 +138,7 @@ class AuthHandler(object): # Check for updated status... try: - self._poll_challenges(chall_update) + self._poll_challenges(chall_update, best_effort) finally: # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) @@ -168,7 +170,7 @@ class AuthHandler(object): return active_achalls def _poll_challenges( - self, chall_update, min_sleep=3, max_rounds=15): + self, chall_update, best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" dom_to_check = set(chall_update.keys()) comp_domains = set() @@ -189,8 +191,14 @@ class AuthHandler(object): chall_update[domain].remove(achall) # We failed some challenges... damage control else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) + if best_effort: + comp_domains.add(domain) + logger.warning( + "Challenge failed for domain %s", + domain) + else: + all_failed_achalls.update( + updated for _, updated in failed_achalls) if all_failed_achalls: _report_failed_challs(all_failed_achalls) diff --git a/certbot/client.py b/certbot/client.py index 61e9db635..5feea662d 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,12 +235,13 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr): + def obtain_certificate_from_csr(self, csr, best_effort=False): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. + :param bool best_effort: Whether or not all authorizations are required :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -258,7 +259,11 @@ class Client(object): logger.debug("CSR: %s", csr) orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr) + authzr = self.auth_handler.handle_authorizations(orderr, best_effort) + if best_effort: + # TODO: check if we passed all authorizations, and if not, + # create a new order and try again, possibly in a loop + pass certr = self.acme.request_issuance( jose.ComparableX509( @@ -313,7 +318,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - certr, chain = self.obtain_certificate_from_csr(csr) + certr, chain = self.obtain_certificate_from_csr(csr, self.config.allow_subset_of_names) return certr, chain, key, csr diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index d424e59ca..ea8b006c4 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -218,7 +218,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1): + def _validate_all(self, unused_1, unused_2): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] self.handler.authzr[dom] = acme_util.gen_authzr( From 11f2f1e576243a255afbcd166b042cab2e2f1c4b Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 13:20:41 -0800 Subject: [PATCH 46/93] remove extra spaces --- certbot/auth_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 662cadc65..47d806b94 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -194,8 +194,8 @@ class AuthHandler(object): if best_effort: comp_domains.add(domain) logger.warning( - "Challenge failed for domain %s", - domain) + "Challenge failed for domain %s", + domain) else: all_failed_achalls.update( updated for _, updated in failed_achalls) From a0e84e65ce9cfc041f341a6bedf36525e49fa1b2 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:29:04 -0800 Subject: [PATCH 47/93] auth_handler tests are happy --- certbot/tests/auth_handler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index ea8b006c4..3633b673d 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -118,7 +118,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) mock_order = mock.MagicMock(authorizations=[authzr]) authzr = self.handler.handle_authorizations(mock_order) From 76a0cbf9c23b9827c4bf94e9a6521a4bf049a466 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:43:12 -0800 Subject: [PATCH 48/93] client tests passing --- certbot/tests/client_test.py | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a9a87b80b..b6cbca367 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -141,16 +141,17 @@ class ClientTest(ClientTestCommon): def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() - self.client.auth_handler.get_authorizations.return_value = [None] + self.client.auth_handler.handle_authorizations.return_value = [None] self.acme.request_issuance.return_value = mock.sentinel.certr self.acme.fetch_chain.return_value = mock.sentinel.chain + self.acme.new_order.return_value = mock.sentinel.orderr def _check_obtain_certificate(self): - self.client.auth_handler.get_authorizations.assert_called_once_with( - self.eg_domains, + self.client.auth_handler.handle_authorizations.assert_called_once_with( + mock.sentinel.orderr, self.config.allow_subset_of_names) - authzr = self.client.auth_handler.get_authorizations() + authzr = self.client.auth_handler.handle_authorizations() self.acme.request_issuance.assert_called_once_with( jose.ComparableX509(OpenSSL.crypto.load_certificate_request( @@ -167,31 +168,19 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, test_csr, - authzr=authzr)) + best_effort=False)) # and that the cert was obtained correctly self._check_obtain_certificate() - # Test for authzr=None - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=None)) - auth_handler.get_authorizations.assert_called_with(self.eg_domains) - # Test for no auth_handler self.client.auth_handler = None self.assertRaises( errors.Error, self.client.obtain_certificate_from_csr, - self.eg_domains, test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) @@ -204,13 +193,10 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="der", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=authzr)) + test_csr)) self.assertEqual(1, mock_get_utility().notification.call_count) @test_util.patch_get_utility() @@ -220,13 +206,10 @@ class ClientTest(ClientTestCommon): test_csr = util.CSR(form="der", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - authzr = auth_handler.get_authorizations(self.eg_domains, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, - self.eg_domains, - test_csr, - authzr=authzr) + test_csr) self.assertEqual(1, mock_get_utility().notification.call_count) @mock.patch("certbot.client.crypto_util") @@ -276,7 +259,7 @@ class ClientTest(ClientTestCommon): identifier=mock.MagicMock( value=domain)))) - self.client.auth_handler.get_authorizations.return_value = authzr + self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): result = self.client.obtain_certificate(self.eg_domains) From 3dfeb483ee853333b11e829ef7d985ebf5ad7269 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:49:23 -0800 Subject: [PATCH 49/93] lint --- certbot/tests/client_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index b6cbca367..570080e3b 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -166,7 +166,6 @@ class ClientTest(ClientTestCommon): mock_logger): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), @@ -191,7 +190,6 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), @@ -204,7 +202,6 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler self.assertRaises( acme_errors.Error, From d6af978472d7519b615e8894903383610ab41269 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 14:52:11 -0800 Subject: [PATCH 50/93] remove if/pass --- certbot/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 5feea662d..65e85a159 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -260,10 +260,8 @@ class Client(object): orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr, best_effort) - if best_effort: - # TODO: check if we passed all authorizations, and if not, - # create a new order and try again, possibly in a loop - pass + # TODO: check if we passed all authorizations, and if not, + # create a new order and try again, possibly in a loop certr = self.acme.request_issuance( jose.ComparableX509( From d29c637bf94cf083ef3dc1648bed3697229b8025 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:36:35 -0800 Subject: [PATCH 51/93] support best_effort --- certbot/client.py | 29 +++++++++++++++++++++-------- certbot/tests/client_test.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 65e85a159..d00055eae 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -235,13 +235,13 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, csr, best_effort=False): + def obtain_certificate_from_csr(self, csr, orderr=None): """Obtain certificate. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. - :param bool best_effort: Whether or not all authorizations are required + :param acme.messages.OrderResource orderr: contains authzrs :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). @@ -258,10 +258,12 @@ class Client(object): logger.debug("CSR: %s", csr) - orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr, best_effort) - # TODO: check if we passed all authorizations, and if not, - # create a new order and try again, possibly in a loop + if orderr is None: + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr) + else: + authzr = orderr.authorizations + certr = self.acme.request_issuance( jose.ComparableX509( @@ -316,9 +318,20 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - certr, chain = self.obtain_certificate_from_csr(csr, self.config.allow_subset_of_names) + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) + auth_domains = set(a.body.identifier.value for a in authzr) + successful_domains = [d for d in domains if d in auth_domains] - return certr, chain, key, csr + if successful_domains != domains: + if not self.config.dry_run: + # TODO: delete keys + pass + return self.obtain_certificate(successful_domains) + else: + certr, chain = self.obtain_certificate_from_csr(csr, orderr) + + return certr, chain, key, csr # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 570080e3b..376bf5a90 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -134,6 +134,7 @@ class ClientTest(ClientTestCommon): self.config.allow_subset_of_names = False self.config.dry_run = False self.eg_domains = ["example.com", "www.example.com"] + self.eg_order = mock.MagicMock(authorizations=[None]) def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] @@ -144,11 +145,11 @@ class ClientTest(ClientTestCommon): self.client.auth_handler.handle_authorizations.return_value = [None] self.acme.request_issuance.return_value = mock.sentinel.certr self.acme.fetch_chain.return_value = mock.sentinel.chain - self.acme.new_order.return_value = mock.sentinel.orderr + self.acme.new_order.return_value = self.eg_order def _check_obtain_certificate(self): self.client.auth_handler.handle_authorizations.assert_called_once_with( - mock.sentinel.orderr, + self.eg_order, self.config.allow_subset_of_names) authzr = self.client.auth_handler.handle_authorizations() @@ -166,15 +167,26 @@ class ClientTest(ClientTestCommon): mock_logger): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, - best_effort=False)) + orderr=orderr)) # and that the cert was obtained correctly self._check_obtain_certificate() + # Test for orderr=None + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr( + test_csr, + orderr=None)) + auth_handler.handle_authorizations.assert_called_with(self.eg_order) + # Test for no auth_handler self.client.auth_handler = None self.assertRaises( @@ -190,11 +202,15 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - test_csr)) + test_csr, + orderr=orderr)) self.assertEqual(1, mock_get_utility().notification.call_count) @test_util.patch_get_utility() @@ -202,11 +218,15 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + orderr = self.acme.new_order(test_csr.data) + authzr = auth_handler.handle_authorizations(orderr, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, - test_csr) + test_csr, + orderr=orderr) self.assertEqual(1, mock_get_utility().notification.call_count) @mock.patch("certbot.client.crypto_util") @@ -256,6 +276,7 @@ class ClientTest(ClientTestCommon): identifier=mock.MagicMock( value=domain)))) + self.eg_order.authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): From 7c073dbcaf6d6789da01734f0a32951b18b3e4d6 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:38:18 -0800 Subject: [PATCH 52/93] lint --- certbot/tests/client_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 376bf5a90..f0ef077b2 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,7 +170,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( @@ -205,7 +204,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( @@ -221,7 +219,6 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - authzr = auth_handler.handle_authorizations(orderr, False) self.assertRaises( acme_errors.Error, self.client.obtain_certificate_from_csr, From 051664a142a8c77121c79cb07307379a63d76874 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:39:30 -0800 Subject: [PATCH 53/93] lint --- certbot/tests/client_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f0ef077b2..f10053616 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -201,7 +201,6 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.side_effect = [acme_errors.Error, mock.sentinel.chain] test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) self.assertEqual( @@ -216,7 +215,6 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() self.acme.fetch_chain.side_effect = acme_errors.Error test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) self.assertRaises( From d5a90c5a6e58a068c0fbd8bf800f9953425a9693 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:43:27 -0800 Subject: [PATCH 54/93] delete key and csr before trying again --- certbot/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index d00055eae..404e1e0d9 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -325,8 +325,8 @@ class Client(object): if successful_domains != domains: if not self.config.dry_run: - # TODO: delete keys - pass + os.remove(key.file) + os.remove(csr.file) return self.obtain_certificate(successful_domains) else: certr, chain = self.obtain_certificate_from_csr(csr, orderr) From 26bcaff85cae0b38280f576ac732bd5a4744d2f8 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 15:59:58 -0800 Subject: [PATCH 55/93] add test for new_order for v2 --- acme/acme/client_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 11516c02f..1ba41cd7d 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -161,6 +161,21 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().register.assert_called_once_with(self.new_reg) mock_client().agree_to_tos.assert_not_called() + def test_new_order_v1(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + + def test_new_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_csr_pem = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_order(mock_csr_pem) + mock_client().new_order.assert_called_once_with(mock_csr_pem) + + + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" From 65d0b9674cb4008e624c1a58cc7c8e4ee91fcdb6 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:01:35 -0800 Subject: [PATCH 56/93] Fix client test --- certbot/tests/client_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f10053616..6b90eab83 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,6 +170,7 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) + auth_handler.handle_authorizations(orderr) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( From a7eadf88629d9cbba611731b9fe555bc0ba5cb84 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:08:46 -0800 Subject: [PATCH 57/93] add new order test for v1 --- acme/acme/client_test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1ba41cd7d..773d59aa2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -161,10 +161,18 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): mock_client().register.assert_called_once_with(self.new_reg) mock_client().agree_to_tos.assert_not_called() - def test_new_order_v1(self): + @mock.patch('OpenSSL.crypto.load_certificate_request') + @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names') + def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names, + mock_load_certificate_request): self.response.json.return_value = DIRECTORY_V1.to_json() + mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com'] + mock_csr_pem = mock.MagicMock() with mock.patch('acme.client.Client') as mock_client: + mock_client().request_domain_challenges.return_value = mock.sentinel.auth client = self._init() + orderr = client.new_order(mock_csr_pem) + self.assertEqual(orderr.authorizations, [mock.sentinel.auth, mock.sentinel.auth]) def test_new_order_v2(self): self.response.json.return_value = DIRECTORY_V2.to_json() From dea43e90b629f629a19e6f865fbac2930421a733 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:11:36 -0800 Subject: [PATCH 58/93] lint --- acme/acme/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 773d59aa2..1b33ca5d7 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -164,7 +164,7 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): @mock.patch('OpenSSL.crypto.load_certificate_request') @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names') def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names, - mock_load_certificate_request): + unused_mock_load_certificate_request): self.response.json.return_value = DIRECTORY_V1.to_json() mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com'] mock_csr_pem = mock.MagicMock() From df50f2d5fa3ddd70cfcd01cf63c305431c577d46 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:12:15 -0800 Subject: [PATCH 59/93] client test --- certbot/tests/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 6b90eab83..ecd77bdeb 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -170,7 +170,7 @@ class ClientTest(ClientTestCommon): auth_handler = self.client.auth_handler orderr = self.acme.new_order(test_csr.data) - auth_handler.handle_authorizations(orderr) + auth_handler.handle_authorizations(orderr, False) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr( From d13a4ed18da3f2a0b8f88076bf15f778337259ea Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 20 Feb 2018 16:50:18 -0800 Subject: [PATCH 60/93] add tests for if partial auth success --- certbot/tests/client_test.py | 47 ++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index ecd77bdeb..f4a8a5c8a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -147,10 +147,13 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.return_value = mock.sentinel.chain self.acme.new_order.return_value = self.eg_order - def _check_obtain_certificate(self): - self.client.auth_handler.handle_authorizations.assert_called_once_with( - self.eg_order, - self.config.allow_subset_of_names) + def _check_obtain_certificate(self, auth_count=1): + if auth_count == 1: + self.client.auth_handler.handle_authorizations.assert_called_once_with( + self.eg_order, + self.config.allow_subset_of_names) + else: + self.assertEqual(self.client.auth_handler.handle_authorizations.call_count, auth_count) authzr = self.client.auth_handler.handle_authorizations() @@ -238,6 +241,21 @@ class ClientTest(ClientTestCommon): mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) + @mock.patch("certbot.client.crypto_util") + @mock.patch("os.remove") + def test_obtain_certificate_partial_success(self, mock_remove, mock_crypto_util): + csr = util.CSR(form="pem", file=mock.sentinel.csr_file, data=CSR_SAN) + key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) + mock_crypto_util.init_save_csr.return_value = csr + mock_crypto_util.init_save_key.return_value = key + + authzr = self._authzr_from_domains(["example.com"]) + self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) + + self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) + self.assertEqual(mock_crypto_util.init_save_csr.call_count, 2) + self.assertEqual(mock_remove.call_count, 2) + @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.acme_crypto_util") def test_obtain_certificate_dry_run(self, mock_acme_crypto, mock_crypto): @@ -255,22 +273,25 @@ class ClientTest(ClientTestCommon): mock_crypto.init_save_key.assert_not_called() mock_crypto.init_save_csr.assert_not_called() - def _test_obtain_certificate_common(self, key, csr): - self._mock_obtain_certificate() - - # return_value is essentially set to (None, None) in - # _mock_obtain_certificate(), which breaks this test. - # Thus fixed by the next line. - + def _authzr_from_domains(self, domains): authzr = [] # domain ordering should not be affected by authorization order - for domain in reversed(self.eg_domains): + for domain in reversed(domains): authzr.append( mock.MagicMock( body=mock.MagicMock( identifier=mock.MagicMock( value=domain)))) + return authzr + + def _test_obtain_certificate_common(self, key, csr, authzr_ret=None, auth_count=1): + self._mock_obtain_certificate() + + # return_value is essentially set to (None, None) in + # _mock_obtain_certificate(), which breaks this test. + # Thus fixed by the next line. + authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr @@ -281,7 +302,7 @@ class ClientTest(ClientTestCommon): self.assertEqual( result, (mock.sentinel.certr, mock.sentinel.chain, key, csr)) - self._check_obtain_certificate() + self._check_obtain_certificate(auth_count) @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') From ea3b78e3c9ff11da1d6919f2c5c36d53623512d3 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 20 Feb 2018 18:53:48 -0800 Subject: [PATCH 61/93] update order object with returned authorizations (#5598) --- certbot/client.py | 6 +++--- certbot/tests/client_test.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 404e1e0d9..dd11f2204 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -261,9 +261,8 @@ class Client(object): if orderr is None: orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr) - else: - authzr = orderr.authorizations - + orderr = orderr.update(authorizations=authzr) + authzr = orderr.authorizations certr = self.acme.request_issuance( jose.ComparableX509( @@ -320,6 +319,7 @@ class Client(object): orderr = self.acme.new_order(csr.data) authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) + orderr = orderr.update(authorizations=authzr) auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f4a8a5c8a..a65341692 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -294,6 +294,7 @@ class ClientTest(ClientTestCommon): authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr + self.eg_order.update().authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): From f1b7017c0c632054c61a231f8cd83db092d063fa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 16 Feb 2018 16:20:45 -0800 Subject: [PATCH 62/93] Finish dropping Python 2.6 and 3.3 support * Undo letsencrypt-auto changes * Remove ordereddict import * Add Python 3.4 tests to replace 3.3 * Add python_requires * update pipstrap --- .travis.yml | 4 + acme/setup.py | 1 + certbot-apache/setup.py | 1 + certbot-compatibility-test/setup.py | 1 + certbot-dns-cloudflare/setup.py | 1 + certbot-dns-cloudxns/setup.py | 1 + certbot-dns-digitalocean/setup.py | 1 + certbot-dns-dnsimple/setup.py | 1 + certbot-dns-dnsmadeeasy/setup.py | 1 + certbot-dns-google/setup.py | 1 + certbot-dns-luadns/setup.py | 1 + certbot-dns-nsone/setup.py | 1 + certbot-dns-rfc2136/setup.py | 1 + certbot-dns-route53/setup.py | 1 + certbot-nginx/setup.py | 1 + certbot/plugins/disco.py | 8 +- certbot/util.py | 8 +- letsencrypt-auto-source/letsencrypt-auto | 124 ++++++++++++------ .../letsencrypt-auto.template | 33 +++-- letsencrypt-auto-source/pieces/pipstrap.py | 91 ++++++++----- letshelp-certbot/setup.py | 1 + setup.py | 1 + 22 files changed, 192 insertions(+), 92 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1077d99d9..42b8d679d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,10 @@ matrix: env: TOXENV=py27-oldest sudo: required services: docker + - python: "3.4" + env: TOXENV=py34 + sudo: required + services: docker - python: "3.6" env: TOXENV=py36 sudo: required diff --git a/acme/setup.py b/acme/setup.py index ba5c8e6fb..1f16f3b99 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -45,6 +45,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index d7c223a0a..86b0c646e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 7e1b059e2..861921ef7 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -34,6 +34,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d619f1872..6db6cc48f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 5d14f3e29..bf337c3d0 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index ce8fedd46..12d55f660 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 06af16759..79c93c942 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 7c0f3ed86..5d0970af1 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index de881ad84..cdfa205aa 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -36,6 +36,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 0d580b7ee..6c0dfb68f 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index c0ba11470..09a4e2cf7 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 5161e7a94..06efc373d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -31,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 09f8a7d52..8bd157166 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 37c477ef6..58f687aea 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -32,6 +32,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 37baf98f7..5a7e07ec0 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -5,6 +5,8 @@ import logging import pkg_resources import six +from collections import OrderedDict + import zope.interface import zope.interface.verify @@ -12,12 +14,6 @@ from certbot import constants from certbot import errors from certbot import interfaces -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) diff --git a/certbot/util.py b/certbot/util.py index b7e60a225..b81799373 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -16,18 +16,14 @@ import stat import subprocess import sys +from collections import OrderedDict + import configargparse from certbot import constants from certbot import errors from certbot import lock -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index aed15a8ef..85659cfad 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -781,11 +781,20 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -956,10 +965,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -1220,6 +1237,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -1253,33 +1271,32 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 0 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), +# Pip has no dependencies, as it vendors everything: +PIP_PACKAGE = [ + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] + + +OTHER_PACKAGES = maybe_argparse + [ # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -1300,12 +1317,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -1315,8 +1333,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -1329,6 +1348,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -1336,17 +1373,24 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # We download and install pip first, then the rest, to avoid the bug + # https://github.com/certbot/certbot/issues/4938. + pip_downloads, other_downloads = [ + [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in packages] + for packages in (PIP_PACKAGE, OTHER_PACKAGES)] + for downloads in (pip_downloads, other_downloads): + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it + # otherwise sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index b3d6ab740..618e8f6bd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -320,11 +320,20 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { @@ -495,10 +504,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 78491b5e3..ed55b37e9 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -23,6 +23,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -56,33 +57,32 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 0 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), +# Pip has no dependencies, as it vendors everything: +PIP_PACKAGE = [ + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] + + +OTHER_PACKAGES = maybe_argparse + [ # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -103,12 +103,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -118,8 +119,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -132,6 +134,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -139,17 +159,24 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # We download and install pip first, then the rest, to avoid the bug + # https://github.com/certbot/certbot/issues/4938. + pip_downloads, other_downloads = [ + [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in packages] + for packages in (PIP_PACKAGE, OTHER_PACKAGES)] + for downloads in (pip_downloads, other_downloads): + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it + # otherwise sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 7c8c39068..b5be07a59 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: System Administrators', diff --git a/setup.py b/setup.py index e3824a7f7..f314449e6 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Console', From f3b23662f1fddedcea702c6ea5b54d5f97d1b0a8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 21 Feb 2018 20:52:04 -0800 Subject: [PATCH 63/93] Don't error immediately on wildcards. (#5600) --- certbot/tests/display/ops_test.py | 4 ++-- certbot/tests/main_test.py | 7 ++----- certbot/util.py | 10 ---------- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 57d82f839..c4f58ba7c 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -300,8 +300,8 @@ class ChooseNamesTest(unittest.TestCase): from certbot.display.ops import get_valid_domains all_valid = ["example.com", "second.example.com", "also.example.com", "under_score.example.com", - "justtld"] - all_invalid = ["öóòps.net", "*.wildcard.com", "uniçodé.com"] + "justtld", "*.wildcard.com"] + all_invalid = ["öóòps.net", "uniçodé.com"] two_valid = ["example.com", "úniçøde.com", "also.example.com"] self.assertEqual(get_valid_domains(all_valid), all_valid) self.assertEqual(get_valid_domains(all_invalid), []) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 518653a53..c31a3fb33 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,3 +1,4 @@ +# coding=utf-8 """Tests for certbot.main.""" # pylint: disable=too-many-lines from __future__ import print_function @@ -939,10 +940,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertRaises(errors.ConfigurationError, self._call, ['-d', (('a' * 50) + '.') * 10]) - # Wildcard - self.assertRaises(errors.ConfigurationError, - self._call, - ['-d', '*.wildcard.tld']) # Bare IP address (this is actually a different error message now) self.assertRaises(errors.ConfigurationError, @@ -1232,7 +1229,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def test_renew_with_bad_domain(self): renewalparams = {'authenticator': 'webroot'} - names = ['*.example.com'] + names = ['uniçodé.com'] self._test_renew_common(renewalparams=renewalparams, error_expected=True, names=names, assert_oc_called=False) diff --git a/certbot/util.py b/certbot/util.py index b7e60a225..70f402a72 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -552,16 +552,6 @@ def enforce_domain_sanity(domain): :returns: The domain cast to `str`, with ASCII-only contents :rtype: str """ - if isinstance(domain, six.text_type): - wildcard_marker = u"*." - else: - wildcard_marker = b"*." - - # Check if there's a wildcard domain - if domain.startswith(wildcard_marker): - raise errors.ConfigurationError( - "Wildcard domains are not supported: {0}".format(domain)) - # Unicode try: if isinstance(domain, six.binary_type): From c3659c300b0720d29939331c1c7c86586fd629ae Mon Sep 17 00:00:00 2001 From: Marcus LaFerrera Date: Thu, 22 Feb 2018 13:09:06 -0500 Subject: [PATCH 64/93] Return str rather than bytes (#5585) * Return str rather than bytes Project id is returned as bytes, which causes issues when constructing the google cloud API url, converting `b'PROJECT_ID'` to `b%27PROJECT_ID%27` causing the request to fail. * Ensure we handle both bytes and str types * project_id should be a str or bytes, not int --- certbot-dns-google/certbot_dns_google/dns_google.py | 5 ++++- certbot-dns-google/certbot_dns_google/dns_google_test.py | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index 37fd6b0de..cea754c06 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -224,4 +224,7 @@ class _GoogleClient(object): if r.status != 200: raise ValueError("Invalid status code: {0}".format(r)) - return content + if isinstance(content, bytes): + return content.decode() + else: + return content diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 85649fc7f..53f84dd6e 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -223,9 +223,13 @@ class GoogleClientTest(unittest.TestCase): response = DummyResponse() response.status = 200 - with mock.patch('httplib2.Http.request', return_value=(response, 1234)): + with mock.patch('httplib2.Http.request', return_value=(response, 'test-test-1')): project_id = _GoogleClient.get_project_id() - self.assertEqual(project_id, 1234) + self.assertEqual(project_id, 'test-test-1') + + with mock.patch('httplib2.Http.request', return_value=(response, b'test-test-1')): + project_id = _GoogleClient.get_project_id() + self.assertEqual(project_id, 'test-test-1') failed_response = DummyResponse() failed_response.status = 404 From 457269b0052f68b7e1b4a75414175a2e22f27ae6 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 22 Feb 2018 10:14:29 -0800 Subject: [PATCH 65/93] Add finalize_order to shim object, update Certbot to use it (#5601) * update order object with returned authorizations * major structure of finalize_order shim refactor * util methods and imports for finalize_order shim refactor * update certbot.tests.client_test.py * extraneous client_test imports * remove correct import * update renewal call * add test for acme.dump_pyopenssl_chain * Add test for certbot.crypto_util.cert_and_chain_from_fullchain * add tests for acme.client and change to fetch chain failure to TimeoutError * s/rytpe/rtype * remove ClientV1 passthrough * dump the wrapped cert * remove dead code * remove the correct dead code * support earlier mock --- acme/acme/client.py | 43 ++++++++++++++-- acme/acme/client_test.py | 71 ++++++++++++++++++++++++- acme/acme/crypto_util.py | 22 ++++++++ acme/acme/crypto_util_test.py | 28 ++++++++++ certbot/client.py | 81 +++++++++-------------------- certbot/crypto_util.py | 25 +++++---- certbot/main.py | 4 +- certbot/renewal.py | 5 +- certbot/tests/client_test.py | 86 +++++++++++-------------------- certbot/tests/crypto_util_test.py | 13 +++++ certbot/tests/util.py | 5 -- 11 files changed, 240 insertions(+), 143 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 1838fab42..97f529aae 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -677,9 +677,6 @@ class BackwardsCompatibleClientV2(object): return getattr(self.client, name) elif name in dir(ClientBase): return getattr(self.client, name) - # temporary, for breaking changes into smaller pieces - elif name in dir(Client): - return getattr(self.client, name) else: raise AttributeError() @@ -723,10 +720,48 @@ class BackwardsCompatibleClientV2(object): authorizations = [] for domain in dnsNames: authorizations.append(self.client.request_domain_challenges(domain)) - return messages.OrderResource(authorizations=authorizations) + return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem) else: return self.client.new_order(csr_pem) + def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ + if self.acme_version == 1: + csr_pem = orderr.csr_pem + certr = self.client.request_issuance( + jose.ComparableX509( + OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)), + orderr.authorizations) + + chain = None + while datetime.datetime.now() < deadline: + try: + chain = self.client.fetch_chain(certr) + break + except errors.Error: + time.sleep(1) + + if chain is None: + raise errors.TimeoutError( + 'Failed to fetch chain. You should not deploy the generated ' + 'certificate, please rerun the command for a new one.') + + cert = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) + chain = crypto_util.dump_pyopenssl_chain(chain) + + return orderr.update(fullchain_pem=(cert + chain)) + else: + return self.client.finalize_order(orderr, deadline) + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1b33ca5d7..acc5193ca 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -8,6 +8,7 @@ from six.moves import http_client # pylint: disable=import-error import josepy as jose import mock +import OpenSSL import requests from acme import challenges @@ -82,6 +83,29 @@ class ClientTestBase(unittest.TestCase): class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" + def setUp(self): + super(BackwardsCompatibleClientV2Test, self).setUp() + # contains a loaded cert + self.certr = messages.CertificateResource( + body=messages_test.CERT) + + loaded = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, CERT_SAN_PEM) + wrapped = jose.ComparableX509(loaded) + self.chain = [wrapped, wrapped] + + self.cert_pem = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped) + + single_chain = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, loaded) + self.chain_pem = single_chain + single_chain + + self.fullchain_pem = self.cert_pem + self.chain_pem + + self.orderr = messages.OrderResource( + csr_pem=CSR_SAN_PEM) + def _init(self): uri = 'http://www.letsencrypt-demo.org/directory' from acme.client import BackwardsCompatibleClientV2 @@ -109,8 +133,6 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client = self._init() self.assertEqual(client.directory, client.client.directory) self.assertEqual(client.key, KEY) - # delete this line once we finish migrating to new API: - self.assertEqual(client.register, client.client.register) self.assertEqual(client.update_registration, client.client.update_registration) self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') @@ -182,7 +204,52 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.new_order(mock_csr_pem) mock_client().new_order.assert_called_once_with(mock_csr_pem) + @mock.patch('acme.client.Client') + def test_finalize_order_v1_success(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + mock_client().fetch_chain.assert_called_once_with(self.certr) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_fetch_chain_error(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + mock_client().fetch_chain.side_effect = [errors.Error, self.chain] + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + self.assertEqual(mock_client().fetch_chain.call_count, 2) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_timeout(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + + deadline = deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + client = self._init() + self.assertRaises(errors.TimeoutError, client.finalize_order, + self.orderr, deadline) + + def test_finalize_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_orderr = mock.MagicMock() + mock_deadline = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.finalize_order(mock_orderr, mock_deadline) + mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline) class ClientTest(ClientTestBase): diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index a986721f0..f13c5109c 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -8,6 +8,8 @@ import socket import sys import OpenSSL +import josepy as jose + from acme import errors @@ -280,3 +282,23 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_pubkey(key) cert.sign(key, "sha256") return cert + +def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): + """Dump certificate chain into a bundle. + + :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in + :class:`josepy.util.ComparableX509`). + + """ + # XXX: returns empty string when no chain is available, which + # shuts up RenewableCert, but might not be the best solution... + + def _dump_cert(cert): + if isinstance(cert, jose.ComparableX509): + # pylint: disable=protected-access + cert = cert.wrapped + return OpenSSL.crypto.dump_certificate(filetype, cert) + + # assumes that OpenSSL.crypto.dump_certificate includes ending + # newline character + return b"".join(_dump_cert(cert) for cert in chain) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 14aaac8b5..e8dd3b20c 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -225,5 +225,33 @@ class MakeCSRTest(unittest.TestCase): self.assertEqual(len(must_staple_exts), 1, "Expected exactly one Must Staple extension") + +class DumpPyopensslChainTest(unittest.TestCase): + """Test for dump_pyopenssl_chain.""" + + @classmethod + def _call(cls, loaded): + # pylint: disable=protected-access + from acme.crypto_util import dump_pyopenssl_chain + return dump_pyopenssl_chain(loaded) + + def test_dump_pyopenssl_chain(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + length = sum( + len(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) + for cert in loaded) + self.assertEqual(len(self._call(loaded)), length) + + def test_dump_pyopenssl_chain_wrapped(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + wrap_func = jose.ComparableX509 + wrapped = [wrap_func(cert) for cert in loaded] + dump_func = OpenSSL.crypto.dump_certificate + length = sum(len(dump_func(OpenSSL.crypto.FILETYPE_PEM, cert)) for cert in loaded) + self.assertEqual(len(self._call(wrapped)), length) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/client.py b/certbot/client.py index dd11f2204..fc3848a5c 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -1,4 +1,5 @@ """Certbot client API.""" +import datetime import logging import os import platform @@ -11,7 +12,6 @@ import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util -from acme import errors as acme_errors from acme import messages import certbot @@ -243,8 +243,7 @@ class Client(object): than `authkey`. :param acme.messages.OrderResource orderr: contains authzrs - :returns: `.CertificateResource` and certificate chain (as - returned by `.fetch_chain`). + :returns: certificate and chain as PEM strings :rtype: tuple """ @@ -264,32 +263,9 @@ class Client(object): orderr = orderr.update(authorizations=authzr) authzr = orderr.authorizations - certr = self.acme.request_issuance( - jose.ComparableX509( - OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr.data)), - authzr) - - notify = zope.component.getUtility(interfaces.IDisplay).notification - retries = 0 - chain = None - - while retries <= 1: - if retries: - notify('Failed to fetch chain, please check your network ' - 'and continue', pause=True) - try: - chain = self.acme.fetch_chain(certr) - break - except acme_errors.Error: - logger.debug('Failed to fetch chain', exc_info=True) - retries += 1 - - if chain is None: - raise acme_errors.Error( - 'Failed to fetch chain. You should not deploy the generated ' - 'certificate, please rerun the command for a new one.') - - return certr, chain + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.acme.finalize_order(orderr, deadline) + return crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. @@ -298,10 +274,9 @@ class Client(object): :param list domains: domains to get a certificate - :returns: `.CertificateResource`, certificate chain (as - returned by `.fetch_chain`), and newly generated private key - (`.util.Key`) and DER-encoded Certificate Signing Request - (`.util.CSR`). + :returns: :returns: certificate as PEM string, chain as PEM string, + newly generated private key (`.util.Key`), and DER-encoded + Certificate Signing Request (`.util.CSR`). :rtype: tuple """ @@ -329,9 +304,9 @@ class Client(object): os.remove(csr.file) return self.obtain_certificate(successful_domains) else: - certr, chain = self.obtain_certificate_from_csr(csr, orderr) + cert, chain = self.obtain_certificate_from_csr(csr, orderr) - return certr, chain, key, csr + return cert, chain, key, csr # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): @@ -350,7 +325,7 @@ class Client(object): be obtained, or None if doing a successful dry run. """ - certr, chain, key, _ = self.obtain_certificate(domains) + cert, chain, key, _ = self.obtain_certificate(domains) if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): @@ -365,19 +340,16 @@ class Client(object): return None else: return storage.RenewableCert.new_lineage( - new_name, OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), - key.pem, crypto_util.dump_pyopenssl_chain(chain), + new_name, cert, + key.pem, chain, self.config) - def save_certificate(self, certr, chain_cert, + def save_certificate(self, cert_pem, chain_pem, cert_path, chain_path, fullchain_path): """Saves the certificate received from the ACME server. - :param certr: ACME "certificate" resource. - :type certr: :class:`acme.messages.Certificate` - - :param list chain_cert: + :param str cert_pem: + :param str chain_pem: :param str cert_path: Candidate path to a certificate. :param str chain_path: Candidate path to a certificate chain. :param str fullchain_path: Candidate path to a full cert chain. @@ -394,8 +366,6 @@ class Client(object): os.path.dirname(path), 0o755, os.geteuid(), self.config.strict_permissions) - cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) cert_file, abs_cert_path = _open_pem_file('cert_path', cert_path) @@ -406,20 +376,15 @@ class Client(object): logger.info("Server issued certificate; certificate written to %s", abs_cert_path) - if not chain_cert: - return abs_cert_path, None, None - else: - chain_pem = crypto_util.dump_pyopenssl_chain(chain_cert) + chain_file, abs_chain_path =\ + _open_pem_file('chain_path', chain_path) + fullchain_file, abs_fullchain_path =\ + _open_pem_file('fullchain_path', fullchain_path) - chain_file, abs_chain_path =\ - _open_pem_file('chain_path', chain_path) - fullchain_file, abs_fullchain_path =\ - _open_pem_file('fullchain_path', fullchain_path) + _save_chain(chain_pem, chain_file) + _save_chain(cert_pem + chain_pem, fullchain_file) - _save_chain(chain_pem, chain_file) - _save_chain(cert_pem + chain_pem, fullchain_file) - - return abs_cert_path, abs_chain_path, abs_fullchain_path + return abs_cert_path, abs_chain_path, abs_fullchain_path def deploy_certificate(self, domains, privkey_path, cert_path, chain_path, fullchain_path): diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 8368855cd..11721cc10 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,7 +14,6 @@ import six import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 -import josepy as jose from acme import crypto_util as acme_crypto_util @@ -367,16 +366,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """ # XXX: returns empty string when no chain is available, which # shuts up RenewableCert, but might not be the best solution... - - def _dump_cert(cert): - if isinstance(cert, jose.ComparableX509): - # pylint: disable=protected-access - cert = cert.wrapped - return OpenSSL.crypto.dump_certificate(filetype, cert) - - # assumes that OpenSSL.crypto.dump_certificate includes ending - # newline character - return b"".join(_dump_cert(cert) for cert in chain) + return acme_crypto_util.dump_pyopenssl_chain(chain, filetype) def notBefore(cert_path): @@ -443,3 +433,16 @@ def sha256sum(filename): with open(filename, 'rb') as f: sha256.update(f.read()) return sha256.hexdigest() + +def cert_and_chain_from_fullchain(fullchain_pem): + """Split fullchain_pem into cert_pem and chain_pem + + :param str fullchain_pem: concatenated cert + chain + + :returns: tuple of string cert_pem and chain_pem + :rtype: tuple + """ + cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, + OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + chain = fullchain_pem[len(cert):] + return (cert, chain) diff --git a/certbot/main.py b/certbot/main.py index d01f68920..eff4c9c8f 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1064,13 +1064,13 @@ def _csr_get_and_save_cert(config, le_client): """ csr, _ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(csr) + cert, chain = le_client.obtain_certificate_from_csr(csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) return None, None cert_path, _, fullchain_path = le_client.save_certificate( - certr, chain, config.cert_path, config.chain_path, config.fullchain_path) + cert, chain, config.cert_path, config.chain_path, config.fullchain_path) return cert_path, fullchain_path def renew_cert(config, plugins, lineage): diff --git a/certbot/renewal.py b/certbot/renewal.py index 024a815cc..ea5d87a5e 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -294,15 +294,12 @@ def renew_cert(config, domains, le_client, lineage): _avoid_invalidating_lineage(config, lineage, original_server) if not domains: domains = lineage.names() - new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) + new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert)) else: prior_version = lineage.latest_common_version() - new_cert = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped) - new_chain = crypto_util.dump_pyopenssl_chain(new_chain) # TODO: Check return value of save_successor lineage.save_successor(prior_version, new_cert, new_key.pem, new_chain, config) lineage.update_all_links_to(lineage.latest_common_version()) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index a65341692..ed9c140e7 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,12 +4,8 @@ import shutil import tempfile import unittest -import josepy as jose -import OpenSSL import mock -from acme import errors as acme_errors - from certbot import account from certbot import errors from certbot import util @@ -134,7 +130,10 @@ class ClientTest(ClientTestCommon): self.config.allow_subset_of_names = False self.config.dry_run = False self.eg_domains = ["example.com", "www.example.com"] - self.eg_order = mock.MagicMock(authorizations=[None]) + self.eg_order = mock.MagicMock( + authorizations=[None], + fullchain_pem=mock.sentinel.fullchain_pem, + csr_pem=mock.sentinel.csr_pem) def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] @@ -143,9 +142,9 @@ class ClientTest(ClientTestCommon): def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() self.client.auth_handler.handle_authorizations.return_value = [None] - self.acme.request_issuance.return_value = mock.sentinel.certr - self.acme.fetch_chain.return_value = mock.sentinel.chain + self.acme.finalize_order.return_value = self.eg_order self.acme.new_order.return_value = self.eg_order + self.eg_order.update.return_value = self.eg_order def _check_obtain_certificate(self, auth_count=1): if auth_count == 1: @@ -155,27 +154,24 @@ class ClientTest(ClientTestCommon): else: self.assertEqual(self.client.auth_handler.handle_authorizations.call_count, auth_count) - authzr = self.client.auth_handler.handle_authorizations() - - self.acme.request_issuance.assert_called_once_with( - jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, CSR_SAN)), - authzr) - - self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) + self.acme.finalize_order.assert_called_once_with( + self.eg_order, mock.ANY) + @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.logger") @test_util.patch_get_utility() def test_obtain_certificate_from_csr(self, unused_mock_get_utility, - mock_logger): + mock_logger, mock_crypto_util): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) orderr = self.acme.new_order(test_csr.data) auth_handler.handle_authorizations(orderr, False) self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, orderr=orderr)) @@ -184,7 +180,7 @@ class ClientTest(ClientTestCommon): # Test for orderr=None self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( test_csr, orderr=None)) @@ -198,41 +194,13 @@ class ClientTest(ClientTestCommon): test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_succeeded( - self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = [acme_errors.Error, - mock.sentinel.chain] - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - - orderr = self.acme.new_order(test_csr.data) - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - test_csr, - orderr=orderr)) - self.assertEqual(1, mock_get_utility().notification.call_count) - - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_failed(self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = acme_errors.Error - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - - orderr = self.acme.new_order(test_csr.data) - self.assertRaises( - acme_errors.Error, - self.client.obtain_certificate_from_csr, - test_csr, - orderr=orderr) - self.assertEqual(1, mock_get_utility().notification.call_count) - @mock.patch("certbot.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): csr = util.CSR(form="pem", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) self._test_obtain_certificate_common(mock.sentinel.key, csr) @@ -240,6 +208,8 @@ class ClientTest(ClientTestCommon): self.config.rsa_key_size, self.config.key_dir) mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) + mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with( + mock.sentinel.fullchain_pem) @mock.patch("certbot.client.crypto_util") @mock.patch("os.remove") @@ -248,6 +218,8 @@ class ClientTest(ClientTestCommon): key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = key + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) authzr = self._authzr_from_domains(["example.com"]) self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) @@ -255,6 +227,7 @@ class ClientTest(ClientTestCommon): self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) self.assertEqual(mock_crypto_util.init_save_csr.call_count, 2) self.assertEqual(mock_remove.call_count, 2) + self.assertEqual(mock_crypto_util.cert_and_chain_from_fullchain.call_count, 1) @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.acme_crypto_util") @@ -263,6 +236,8 @@ class ClientTest(ClientTestCommon): mock_acme_crypto.make_csr.return_value = CSR_SAN mock_crypto.make_key.return_value = mock.sentinel.key_pem key = util.Key(file=None, pem=mock.sentinel.key_pem) + mock_crypto.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) self.client.config.dry_run = True self._test_obtain_certificate_common(key, csr) @@ -272,6 +247,7 @@ class ClientTest(ClientTestCommon): mock.sentinel.key_pem, self.eg_domains, self.config.must_staple) mock_crypto.init_save_key.assert_not_called() mock_crypto.init_save_csr.assert_not_called() + self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1) def _authzr_from_domains(self, domains): authzr = [] @@ -294,7 +270,6 @@ class ClientTest(ClientTestCommon): authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) self.eg_order.authorizations = authzr - self.eg_order.update().authorizations = authzr self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): @@ -302,13 +277,12 @@ class ClientTest(ClientTestCommon): self.assertEqual( result, - (mock.sentinel.certr, mock.sentinel.chain, key, csr)) + (mock.sentinel.cert, mock.sentinel.chain, key, csr)) self._check_obtain_certificate(auth_count) @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') - @mock.patch('OpenSSL.crypto.dump_certificate') - def test_obtain_and_enroll_certificate(self, mock_dump_certificate, + def test_obtain_and_enroll_certificate(self, mock_storage, mock_obtain_certificate): domains = ["example.com", "www.example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), @@ -324,7 +298,6 @@ class ClientTest(ClientTestCommon): self.assertFalse(self.client.obtain_and_enroll_certificate(domains, None)) self.assertTrue(mock_storage.call_count == 2) - self.assertTrue(mock_dump_certificate.call_count == 2) @mock.patch("certbot.cli.helpful_parser") def test_save_certificate(self, mock_parser): @@ -333,9 +306,8 @@ class ClientTest(ClientTestCommon): tmp_path = tempfile.mkdtemp() os.chmod(tmp_path, 0o755) # TODO: really?? - certr = mock.MagicMock(body=test_util.load_comparable_cert(certs[0])) - chain_cert = [test_util.load_comparable_cert(certs[0]), - test_util.load_comparable_cert(certs[1])] + cert_pem = test_util.load_vector(certs[0]) + chain_pem = (test_util.load_vector(certs[0]) + test_util.load_vector(certs[1])) candidate_cert_path = os.path.join(tmp_path, "certs", "cert_512.pem") candidate_chain_path = os.path.join(tmp_path, "chains", "chain.pem") candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") @@ -345,7 +317,7 @@ class ClientTest(ClientTestCommon): "--fullchain-path", candidate_fullchain_path] cert_path, chain_path, fullchain_path = self.client.save_certificate( - certr, chain_cert, candidate_cert_path, candidate_chain_path, + cert_pem, chain_pem, candidate_cert_path, candidate_chain_path, candidate_fullchain_path) self.assertEqual(os.path.dirname(cert_path), diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index f0e2c017e..00303fab3 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -373,5 +373,18 @@ class Sha256sumTest(unittest.TestCase): '914ffed8daf9e2c99d90ac95c77d54f32cbd556672facac380f0c063498df84e') +class CertAndChainFromFullchainTest(unittest.TestCase): + """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" + + def test_cert_and_chain_from_fullchain(self): + cert_pem = CERT + chain_pem = CERT + SS_CERT + fullchain_pem = cert_pem + chain_pem + from certbot.crypto_util import cert_and_chain_from_fullchain + cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem) + self.assertEqual(cert_out, cert_pem) + self.assertEqual(chain_out, chain_pem) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 60d8d6084..8434d11de 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -57,11 +57,6 @@ def load_cert(*names): return OpenSSL.crypto.load_certificate(loader, load_vector(*names)) -def load_comparable_cert(*names): - """Load ComparableX509 cert.""" - return jose.ComparableX509(load_cert(*names)) - - def load_csr(*names): """Load certificate request.""" loader = _guess_loader( From 990b211a76efb5a376cc4076bbbc2c2f0a2b3f2b Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 22 Feb 2018 12:33:55 -0800 Subject: [PATCH 66/93] Remove extra `:returns:` (#5611) --- certbot/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/client.py b/certbot/client.py index fc3848a5c..2d7288ce3 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -274,7 +274,7 @@ class Client(object): :param list domains: domains to get a certificate - :returns: :returns: certificate as PEM string, chain as PEM string, + :returns: certificate as PEM string, chain as PEM string, newly generated private key (`.util.Key`), and DER-encoded Certificate Signing Request (`.util.CSR`). :rtype: tuple From 1e46d26ac3511ca92e14c98c823a82286142ebb6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 22 Feb 2018 16:28:50 -0800 Subject: [PATCH 67/93] Fix ACMEv2 issues (#5612) * Add post wrapper to automatically add acme_version * Add uri to authzr. * Only add kid when account is set. * Add content_type when downloading certificate. * Only save new_authz URL when it exists. * Handle combinations in ACMEv1 and ACMEv2. * Add tests for ACMEv2 "combinations". --- acme/acme/client.py | 61 ++++++++++++-------- certbot/account.py | 17 ++++-- certbot/auth_handler.py | 12 +++- certbot/client.py | 4 +- certbot/tests/auth_handler_test.py | 93 ++++++++++++++++++++++++------ 5 files changed, 134 insertions(+), 53 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 97f529aae..9854aae31 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -70,7 +70,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes terms_of_service=terms_of_service) def _send_recv_regr(self, regr, body): - response = self.net.post(regr.uri, body, acme_version=self.acme_version) + response = self._post(regr.uri, body) # TODO: Boulder returns httplib.ACCEPTED #assert response.status_code == httplib.OK @@ -82,6 +82,13 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response, uri=regr.uri, terms_of_service=regr.terms_of_service) + def _post(self, *args, **kwargs): + """Wrapper around self.net.post that adds the acme_version. + + """ + kwargs.setdefault('acme_version', self.acme_version) + return self.net.post(*args, **kwargs) + def update_registration(self, regr, update=None): """Update registration. @@ -143,8 +150,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self.net.post(challb.uri, response, - acme_version=self.acme_version) + response = self._post(challb.uri, response) try: authzr_uri = response.links['up']['url'] except KeyError: @@ -216,12 +222,11 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .ClientError: If revocation is unsuccessful. """ - response = self.net.post(self.directory[messages.Revocation], + response = self._post(self.directory[messages.Revocation], messages.Revocation( certificate=cert, reason=rsn), - content_type=None, - acme_version=self.acme_version) + content_type=None) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') @@ -271,8 +276,7 @@ class Client(ClientBase): """ new_reg = messages.NewRegistration() if new_reg is None else new_reg - response = self.net.post(self.directory[new_reg], new_reg, - acme_version=1) + response = self._post(self.directory[new_reg], new_reg) # TODO: handle errors assert response.status_code == http_client.CREATED @@ -308,8 +312,7 @@ class Client(ClientBase): if new_authzr_uri is not None: logger.debug("request_challenges with new_authzr_uri deprecated.") new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz, new_authz, - acme_version=1) + response = self._post(self.directory.new_authz, new_authz) # TODO: handle errors assert response.status_code == http_client.CREATED return self._authzr_from_response(response, identifier) @@ -351,12 +354,11 @@ class Client(ClientBase): req = messages.CertificateRequest(csr=csr) content_type = DER_CONTENT_TYPE # TODO: add 'cert_type 'argument - response = self.net.post( + response = self._post( self.directory.new_cert, req, content_type=content_type, - headers={'Accept': content_type}, - acme_version=1) + headers={'Accept': content_type}) cert_chain_uri = response.links.get('up', {}).get('url') @@ -552,8 +554,7 @@ class ClientV2(ClientBase): :returns: Registration Resource. :rtype: `.RegistrationResource` """ - response = self.net.post(self.directory['newAccount'], new_account, - acme_version=2) + response = self._post(self.directory['newAccount'], new_account) # "Instance of 'Field' has no key/contact member" bug: # pylint: disable=no-member regr = self._regr_from_response(response) @@ -577,11 +578,11 @@ class ClientV2(ClientBase): identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=name)) order = messages.NewOrder(identifiers=identifiers) - response = self.net.post(self.directory['newOrder'], order) + response = self._post(self.directory['newOrder'], order) body = messages.Order.from_json(response.json()) authorizations = [] for url in body.authorizations: - authorizations.append(self._authzr_from_response(self.net.get(url))) + authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) return messages.OrderResource( body=body, uri=response.headers.get('Location'), @@ -643,7 +644,7 @@ class ClientV2(ClientBase): csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) - self.net.post(orderr.body.finalize, wrapped_csr) + self._post(orderr.body.finalize, wrapped_csr) while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) @@ -651,17 +652,29 @@ class ClientV2(ClientBase): if body.error is not None: raise errors.IssuanceError(body.error) if body.certificate is not None: - certificate_response = self.net.get(body.certificate).text + certificate_response = self.net.get(body.certificate, + content_type=DER_CONTENT_TYPE).text return orderr.update(body=body, fullchain_pem=certificate_response) raise errors.TimeoutError() class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but - supports V1 servers. + supports V1 servers. - :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint - :ivar .ClientBase client: either Client or ClientV2 + .. note:: While this class handles the majority of the differences + between versions of the ACME protocol, if you need to support an + ACME server based on version 3 or older of the IETF ACME draft + that uses combinations in authorizations (or lack thereof) to + signal that the client needs to complete something other than + any single challenge in the authorization to make it valid, the + user of this class needs to understand and handle these + differences themselves. This does not apply to either of Let's + Encrypt's endpoints where successfully completing any challenge + in an authorization will make it valid. + + :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint + :ivar .ClientBase client: either Client or ClientV2 """ def __init__(self, net, key, server): @@ -829,7 +842,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes } if acme_version == 2: kwargs["url"] = url - kwargs["kid"] = self.account["uri"] + # newAccount and revokeCert work without the kid + if self.account is not None: + kwargs["kid"] = self.account["uri"] kwargs["key"] = self.key # pylint: disable=star-args return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2) diff --git a/certbot/account.py b/certbot/account.py index 41e980097..70d9a7fc3 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -223,12 +223,17 @@ class AccountFileStorage(interfaces.AccountStorage): try: with open(self._regr_path(account_dir_path), "w") as regr_file: regr = account.regr - with_uri = RegistrationResourceWithNewAuthzrURI( - new_authzr_uri=acme.directory.new_authz, - body=regr.body, - uri=regr.uri, - terms_of_service=regr.terms_of_service) - regr_file.write(with_uri.json_dumps()) + # If we have a value for new-authz, save it for forwards + # compatibility with older versions of Certbot. If we don't + # have a value for new-authz, this is an ACMEv2 directory where + # an older version of Certbot won't work anyway. + if hasattr(acme.directory, "new-authz"): + regr = RegistrationResourceWithNewAuthzrURI( + new_authzr_uri=acme.directory.new_authz, + body=regr.body, + uri=regr.uri, + terms_of_service=regr.terms_of_service) + regr_file.write(regr.json_dumps()) if not regr_only: with util.safe_open(self._key_path(account_dir_path), "w", chmod=0o400) as key_file: diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 47d806b94..9cc10d4b4 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -24,7 +24,7 @@ class AuthHandler(object): :class:`~acme.challenges.Challenge` types :type auth: :class:`certbot.interfaces.IAuthenticator` - :ivar acme.client.Client acme: ACME client API. + :ivar acme.client.BackwardsCompatibleClientV2 acme: ACME client API. :ivar account: Client's Account :type account: :class:`certbot.account.Account` @@ -100,10 +100,16 @@ class AuthHandler(object): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") for dom in domains: + dom_challenges = self.authzr[dom].body.challenges + if self.acme.acme_version == 1: + combinations = self.authzr[dom].body.combinations + else: + combinations = tuple((i,) for i in range(len(dom_challenges))) + path = gen_challenge_path( - self.authzr[dom].body.challenges, + dom_challenges, self._get_chall_pref(dom), - self.authzr[dom].body.combinations) + combinations) dom_achalls = self._challenge_factory( dom, path) diff --git a/certbot/client.py b/certbot/client.py index 2d7288ce3..0f4fa760d 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -212,8 +212,8 @@ class Client(object): :ivar .IAuthenticator auth: Prepared (`.IAuthenticator.prepare`) authenticator that can solve ACME challenges. :ivar .IInstaller installer: Installer. - :ivar acme.client.Client acme: Optional ACME client API handle. - You might already have one from `register`. + :ivar acme.client.BackwardsCompatibleClientV2 acme: Optional ACME + client API handle. You might already have one from `register`. """ diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 3633b673d..394002206 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -81,6 +81,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_account = mock.Mock(key=util.Key("file_path", "PEM")) self.mock_net = mock.MagicMock(spec=acme_client.Client) + self.mock_net.acme_version = 1 self.handler = AuthHandler( self.mock_auth, self.mock_net, self.mock_account, []) @@ -90,13 +91,13 @@ class HandleAuthorizationsTest(unittest.TestCase): def tearDown(self): logging.disable(logging.NOTSET) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1(self, mock_poll): - mock_poll.side_effect = self._validate_all - - authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES) + def _test_name1_tls_sni_01_1_common(self, combos): + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos) mock_order = mock.MagicMock(authorizations=[authzr]) - authzr = self.handler.handle_authorizations(mock_order) + + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) @@ -112,8 +113,15 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) + def test_name1_tls_sni_01_1_acme_1(self): + self._test_name1_tls_sni_01_1_common(combos=True) + + def test_name1_tls_sni_01_1_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name1_tls_sni_01_1_common(combos=False) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll): + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self, mock_poll): mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) @@ -138,17 +146,43 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name3_tls_sni_01_3(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) - + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self, mock_poll): + self.mock_net.acme_version = 2 mock_poll.side_effect = self._validate_all + self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) + self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) + + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) + + self.assertEqual(self.mock_net.answer_challenge.call_count, 1) + + self.assertEqual(mock_poll.call_count, 1) + chall_update = mock_poll.call_args[0][0] + self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(len(chall_update.values()), 1) + + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + cleaned_up_achalls = self.mock_auth.cleanup.call_args[0][0] + self.assertEqual(len(cleaned_up_achalls), 1) + self.assertEqual(cleaned_up_achalls[0].typ, "tls-sni-01") + + # Length of authorizations list + self.assertEqual(len(authzr), 1) + + def _test_name3_tls_sni_01_3_common(self, combos): + self.mock_net.request_domain_challenges.side_effect = functools.partial( + gen_dom_authzr, challs=acme_util.CHALLENGES, combos=combos) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - authzr = self.handler.handle_authorizations(mock_order) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -167,6 +201,13 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 3) + def test_name3_tls_sni_01_3_common_acme_1(self): + self._test_name3_tls_sni_01_3_common(combos=True) + + def test_name3_tls_sni_01_3_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name3_tls_sni_01_3_common(combos=False) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_debug_challenges(self, mock_poll): zope.component.provideUtility( @@ -194,30 +235,44 @@ class HandleAuthorizationsTest(unittest.TestCase): mock_order = mock.MagicMock(authorizations=[]) self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_preferred_challenge_choice(self, mock_poll): - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + def _test_preferred_challenge_choice_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - self.handler.handle_authorizations(mock_order) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") - def test_preferred_challenges_not_supported(self): - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + def test_preferred_challenge_choice_common_acme_1(self): + self._test_preferred_challenge_choice_common(combos=True) + + def test_preferred_challenge_choice_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenge_choice_common(combos=False) + + def _test_preferred_challenges_not_supported_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] mock_order = mock.MagicMock(authorizations=authzrs) self.handler.pref_challs.append(challenges.HTTP01.typ) self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def test_preferred_challenges_not_supported_acme_1(self): + self._test_preferred_challenges_not_supported_common(combos=True) + + def test_preferred_challenges_not_supported_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenges_not_supported_common(combos=False) + def _validate_all(self, unused_1, unused_2): for dom in six.iterkeys(self.handler.authzr): azr = self.handler.authzr[dom] From f3a0deba840f1c6bc1510ee71a0e70035fa8488c Mon Sep 17 00:00:00 2001 From: Nick Bebout Date: Fri, 23 Feb 2018 15:26:11 -0600 Subject: [PATCH 68/93] Remove min version of setuptools (#5617) --- acme/setup.py | 4 +--- certbot-apache/setup.py | 4 +--- certbot-dns-cloudflare/setup.py | 4 +--- certbot-dns-cloudxns/setup.py | 4 +--- certbot-dns-digitalocean/setup.py | 4 +--- certbot-dns-dnsimple/setup.py | 4 +--- certbot-dns-dnsmadeeasy/setup.py | 4 +--- certbot-dns-google/setup.py | 4 +--- certbot-dns-luadns/setup.py | 4 +--- certbot-dns-nsone/setup.py | 4 +--- certbot-dns-rfc2136/setup.py | 4 +--- certbot-dns-route53/setup.py | 4 +--- certbot-nginx/setup.py | 4 +--- setup.py | 4 +--- 14 files changed, 14 insertions(+), 42 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index ce426cf74..51bbf0f71 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -19,9 +19,7 @@ install_requires = [ 'pyrfc3339', 'pytz', 'requests[security]>=2.4.1', # security extras added in 2.4.1 - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible ] diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 38f41e9f1..76d7f5ca5 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'mock', 'python-augeas', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 612e7259f..e1b84d1be 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'cloudflare>=1.5.1', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3157400c6..53ceb58ea 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1a68400fa..3330bdd67 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'mock', 'python-digitalocean>=1.11', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six', 'zope.interface', ] diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 35de47308..00a3c032a 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a946d00a4..36119ade0 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8585fc848..10fccd1ea 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -15,9 +15,7 @@ install_requires = [ 'mock', # for oauth2client.service_account.ServiceAccountCredentials 'oauth2client>=2.0', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', # already a dependency of google-api-python-client, but added for consistency 'httplib2' diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4fec37e29..b094e1818 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index dca9ebf27..e777d821b 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bfa72b50b..6fc6dca73 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dnspython', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8df687972..0fbeab31b 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -10,9 +10,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'boto3', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 152f77de8..a84efe2c3 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -13,9 +13,7 @@ install_requires = [ 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] diff --git a/setup.py b/setup.py index 47b5b0b2c..736ef467f 100644 --- a/setup.py +++ b/setup.py @@ -46,9 +46,7 @@ install_requires = [ 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] From 57bdc590dfcbe44da5565e8369997af633debadc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 26 Feb 2018 16:27:38 -0800 Subject: [PATCH 69/93] Add DNS Dockerfiles --- certbot-dns-cloudflare/Dockerfile | 5 +++++ certbot-dns-cloudxns/Dockerfile | 5 +++++ certbot-dns-digitalocean/Dockerfile | 5 +++++ certbot-dns-dnsimple/Dockerfile | 5 +++++ certbot-dns-dnsmadeeasy/Dockerfile | 5 +++++ certbot-dns-google/Dockerfile | 5 +++++ certbot-dns-luadns/Dockerfile | 5 +++++ certbot-dns-nsone/Dockerfile | 5 +++++ certbot-dns-rfc2136/Dockerfile | 5 +++++ certbot-dns-route53/Dockerfile | 5 +++++ 10 files changed, 50 insertions(+) create mode 100644 certbot-dns-cloudflare/Dockerfile create mode 100644 certbot-dns-cloudxns/Dockerfile create mode 100644 certbot-dns-digitalocean/Dockerfile create mode 100644 certbot-dns-dnsimple/Dockerfile create mode 100644 certbot-dns-dnsmadeeasy/Dockerfile create mode 100644 certbot-dns-google/Dockerfile create mode 100644 certbot-dns-luadns/Dockerfile create mode 100644 certbot-dns-nsone/Dockerfile create mode 100644 certbot-dns-rfc2136/Dockerfile create mode 100644 certbot-dns-route53/Dockerfile diff --git a/certbot-dns-cloudflare/Dockerfile b/certbot-dns-cloudflare/Dockerfile new file mode 100644 index 000000000..27dcc8751 --- /dev/null +++ b/certbot-dns-cloudflare/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-cloudflare + +RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare diff --git a/certbot-dns-cloudxns/Dockerfile b/certbot-dns-cloudxns/Dockerfile new file mode 100644 index 000000000..cc84ea65b --- /dev/null +++ b/certbot-dns-cloudxns/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-cloudxns + +RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns diff --git a/certbot-dns-digitalocean/Dockerfile b/certbot-dns-digitalocean/Dockerfile new file mode 100644 index 000000000..8bdd0619f --- /dev/null +++ b/certbot-dns-digitalocean/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-digitalocean + +RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean diff --git a/certbot-dns-dnsimple/Dockerfile b/certbot-dns-dnsimple/Dockerfile new file mode 100644 index 000000000..38d2be80e --- /dev/null +++ b/certbot-dns-dnsimple/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-dnsimple + +RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple diff --git a/certbot-dns-dnsmadeeasy/Dockerfile b/certbot-dns-dnsmadeeasy/Dockerfile new file mode 100644 index 000000000..ff7936925 --- /dev/null +++ b/certbot-dns-dnsmadeeasy/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-dnsmadeeasy + +RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy diff --git a/certbot-dns-google/Dockerfile b/certbot-dns-google/Dockerfile new file mode 100644 index 000000000..4a258d0ee --- /dev/null +++ b/certbot-dns-google/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-google + +RUN pip install --no-cache-dir --editable src/certbot-dns-google diff --git a/certbot-dns-luadns/Dockerfile b/certbot-dns-luadns/Dockerfile new file mode 100644 index 000000000..6efb4d777 --- /dev/null +++ b/certbot-dns-luadns/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-luadns + +RUN pip install --no-cache-dir --editable src/certbot-dns-luadns diff --git a/certbot-dns-nsone/Dockerfile b/certbot-dns-nsone/Dockerfile new file mode 100644 index 000000000..88fc13c57 --- /dev/null +++ b/certbot-dns-nsone/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-nsone + +RUN pip install --no-cache-dir --editable src/certbot-dns-nsone diff --git a/certbot-dns-rfc2136/Dockerfile b/certbot-dns-rfc2136/Dockerfile new file mode 100644 index 000000000..1b8feb2f8 --- /dev/null +++ b/certbot-dns-rfc2136/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-rfc2136 + +RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136 diff --git a/certbot-dns-route53/Dockerfile b/certbot-dns-route53/Dockerfile new file mode 100644 index 000000000..a1b8d6caf --- /dev/null +++ b/certbot-dns-route53/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-route53 + +RUN pip install --no-cache-dir --editable src/certbot-dns-route53 From 6f86267a26c9b748bd90113fe62157a9a455cdd2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 12:42:13 -0800 Subject: [PATCH 70/93] Fix revocation in ACMEv2 (#5626) * Allow revoke to pass in a url * Add revocation support to ACMEv2. * Provide regr for account based revocation. * Add revoke wrapper to BackwardsCompat client --- acme/acme/client.py | 52 +++++++++++++++++++++++++++++++++++----- acme/acme/client_test.py | 26 +++++++++++++++++--- certbot/main.py | 4 ++-- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 9854aae31..e3f6e845d 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -211,7 +211,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response, authzr.body.identifier, authzr.uri) return updated_authzr, response - def revoke(self, cert, rsn): + def _revoke(self, cert, rsn, url): """Revoke certificate. :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in @@ -219,14 +219,16 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :param int rsn: Reason code for certificate revocation. + :param str url: ACME URL to post to + :raises .ClientError: If revocation is unsuccessful. """ - response = self._post(self.directory[messages.Revocation], - messages.Revocation( - certificate=cert, - reason=rsn), - content_type=None) + response = self._post(url, + messages.Revocation( + certificate=cert, + reason=rsn), + content_type=None) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') @@ -528,6 +530,18 @@ class Client(ClientBase): "Recursion limit reached. Didn't get {0}".format(uri)) return chain + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self._revoke(cert, rsn, self.directory[messages.Revocation]) class ClientV2(ClientBase): @@ -657,6 +671,19 @@ class ClientV2(ClientBase): return orderr.update(body=body, fullchain_pem=certificate_response) raise errors.TimeoutError() + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self._revoke(cert, rsn, self.directory['revokeCert']) + class BackwardsCompatibleClientV2(object): """ACME client wrapper that tends towards V2-style calls, but @@ -775,6 +802,19 @@ class BackwardsCompatibleClientV2(object): else: return self.client.finalize_order(orderr, deadline) + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self.client.revoke(cert, rsn) + def _acme_version_from_directory(self, directory): if hasattr(directory, 'newNonce'): return 2 diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index acc5193ca..1e4db2884 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -40,6 +40,7 @@ DIRECTORY_V2 = messages.Directory({ 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce', 'newOrder': 'https://www.letsencrypt-demo.org/acme/new-order', + 'revokeCert': 'https://www.letsencrypt-demo.org/acme/revoke-cert', }) @@ -79,6 +80,9 @@ class ClientTestBase(unittest.TestCase): self.authzr = messages.AuthorizationResource( body=self.authz, uri=authzr_uri) + # Reason code for revocation + self.rsn = 1 + class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" @@ -251,6 +255,19 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): client.finalize_order(mock_orderr, mock_deadline) mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline) + def test_revoke(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" @@ -271,9 +288,6 @@ class ClientTest(ClientTestBase): uri='https://www.letsencrypt-demo.org/acme/cert/1', cert_chain_uri='https://www.letsencrypt-demo.org/ca') - # Reason code for revocation - self.rsn = 1 - from acme.client import Client self.client = Client( directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) @@ -752,6 +766,12 @@ class ClientV2Test(ClientTestBase): deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) self.assertRaises(errors.TimeoutError, self.client.finalize_order, self.orderr, deadline) + def test_revoke(self): + self.client.revoke(messages_test.CERT, self.rsn) + self.net.post.assert_called_once_with( + self.directory["revokeCert"], mock.ANY, content_type=None, + acme_version=2) + class MockJSONDeSerializable(jose.JSONDeSerializable): # pylint: disable=missing-docstring diff --git a/certbot/main.py b/certbot/main.py index 33c7730c6..7be852e83 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -982,11 +982,11 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config config.cert_path[0], config.key_path[0]) crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) key = jose.JWK.load(config.key_path[1]) + acme = client.acme_from_config_key(config, key) else: # revocation by account key logger.debug("Revoking %s using Account Key", config.cert_path[0]) acc, _ = _determine_account(config) - key = acc.key - acme = client.acme_from_config_key(config, key) + acme = client.acme_from_config_key(config, acc.key, acc.regr) cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] logger.debug("Reason code for revocation: %s", config.reason) From b18696b6a0967ddb609aeb5841b93e6214099080 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 16:47:43 -0800 Subject: [PATCH 71/93] Don't run tests with Python 2.6 (#5627) * Don't run tests with Python 2.6. * Revert "Don't run tests with Python 2.6." This reverts commit 4a9d778cca62ae2bec4cf060726e88f1fd66f374. * Revert changes to auto_test.py. --- letsencrypt-auto-source/tests/auto_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 8c2bfc079..d187452a1 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -287,8 +287,8 @@ class AutoTests(TestCase): self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: - self.assertIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertTrue('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This # conveniently sets us up to test the next 2 cases. @@ -296,8 +296,8 @@ class AutoTests(TestCase): # Test when neither phase-1 upgrade nor phase-2 upgrade is # needed (probably a common case): out, err = run_letsencrypt_auto() - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertNotIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertFalse('Creating virtual environment...' in out) def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" @@ -312,8 +312,8 @@ class AutoTests(TestCase): # Create venv saving the correct bootstrap script version out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: bootstrap_version = f.read() @@ -329,8 +329,8 @@ class AutoTests(TestCase): out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) - self.assertNotIn('Upgrading certbot-auto ', out) - self.assertIn('Creating virtual environment...', out) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) def test_openssl_failure(self): """Make sure we stop if the openssl signature check fails.""" From a39d2fe55b760707718dcd8225b7dc42dcef4c9c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 27 Feb 2018 18:05:33 -0800 Subject: [PATCH 72/93] Fix wildcard issuance (#5620) * Add is_wildcard_domain to certbot.util. * Error with --allow-subset-of-names and wildcards. * Fix issue preventing wildcard cert issuance. * Kill assumption domain is unique in auth_handler * fix typo and add test * update comments --- certbot/auth_handler.py | 158 ++++++++++++++++------------- certbot/cli.py | 5 + certbot/client.py | 7 +- certbot/tests/auth_handler_test.py | 80 +++++++-------- certbot/tests/cli_test.py | 4 + certbot/tests/client_test.py | 1 + certbot/tests/util_test.py | 20 ++++ certbot/util.py | 18 ++++ 8 files changed, 180 insertions(+), 113 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 9cc10d4b4..2b38e4af5 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -1,4 +1,5 @@ """ACME AuthHandler.""" +import collections import logging import time @@ -17,6 +18,10 @@ from certbot import interfaces logger = logging.getLogger(__name__) +AnnotatedAuthzr = collections.namedtuple("AnnotatedAuthzr", ["authzr", "achalls"]) +"""Stores an authorization resource and its active annotated challenges.""" + + class AuthHandler(object): """ACME Authorization Handler for a client. @@ -29,10 +34,8 @@ class AuthHandler(object): :ivar account: Client's Account :type account: :class:`certbot.account.Account` - :ivar dict authzr: ACME Authorization Resource dict where keys are domains - and values are :class:`acme.messages.AuthorizationResource` - :ivar list achalls: DV challenges in the form of - :class:`certbot.achallenges.AnnotatedChallenge` + :ivar aauthzrs: ACME Authorization Resources and their active challenges + :type aauthzrs: `list` of `AnnotatedAuthzr` :ivar list pref_challs: sorted user specified preferred challenges type strings with the most preferred challenge listed first @@ -42,12 +45,9 @@ class AuthHandler(object): self.acme = acme self.account = account - self.authzr = dict() + self.aauthzrs = [] self.pref_challs = pref_challs - # List must be used to keep responses straight. - self.achalls = [] - def handle_authorizations(self, orderr, best_effort=False): """Retrieve all authorizations for challenges. @@ -63,17 +63,15 @@ class AuthHandler(object): authorizations """ - authzrs = orderr.authorizations - for authzr in authzrs: - self.authzr[authzr.body.identifier.value] = authzr - domains = self.authzr.keys() + for authzr in orderr.authorizations: + self.aauthzrs.append(AnnotatedAuthzr(authzr, [])) - self._choose_challenges(domains) + self._choose_challenges() config = zope.component.getUtility(interfaces.IConfig) notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... - while self.achalls: + while self._has_challenges(): resp = self._solve_challenges() logger.info("Waiting for verification...") if config.debug_challenges: @@ -87,8 +85,8 @@ class AuthHandler(object): self.verify_authzr_complete() # Only return valid authorizations - retVal = [authzr for authzr in self.authzr.values() - if authzr.body.status == messages.STATUS_VALID] + retVal = [aauthzr.authzr for aauthzr in self.aauthzrs + if aauthzr.authzr.body.status == messages.STATUS_VALID] if not retVal: raise errors.AuthorizationError( @@ -96,41 +94,54 @@ class AuthHandler(object): return retVal - def _choose_challenges(self, domains): + def _choose_challenges(self): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") - for dom in domains: - dom_challenges = self.authzr[dom].body.challenges + for aauthzr in self.aauthzrs: + aauthzr_challenges = aauthzr.authzr.body.challenges if self.acme.acme_version == 1: - combinations = self.authzr[dom].body.combinations + combinations = aauthzr.authzr.body.combinations else: - combinations = tuple((i,) for i in range(len(dom_challenges))) + combinations = tuple((i,) for i in range(len(aauthzr_challenges))) path = gen_challenge_path( - dom_challenges, - self._get_chall_pref(dom), + aauthzr_challenges, + self._get_chall_pref(aauthzr.authzr.body.identifier.value), combinations) - dom_achalls = self._challenge_factory( - dom, path) - self.achalls.extend(dom_achalls) + aauthzr_achalls = self._challenge_factory( + aauthzr.authzr, path) + aauthzr.achalls.extend(aauthzr_achalls) + + def _has_challenges(self): + """Do we have any challenges to perform?""" + return any(aauthzr.achalls for aauthzr in self.aauthzrs) def _solve_challenges(self): """Get Responses for challenges from authenticators.""" resp = [] + all_achalls = self._get_all_achalls() with error_handler.ErrorHandler(self._cleanup_challenges): try: - if self.achalls: - resp = self.auth.perform(self.achalls) + if all_achalls: + resp = self.auth.perform(all_achalls) except errors.AuthorizationError: logger.critical("Failure in setting up challenges.") logger.info("Attempting to clean up outstanding challenges...") raise - assert len(resp) == len(self.achalls) + assert len(resp) == len(all_achalls) return resp + def _get_all_achalls(self): + """Return all active challenges.""" + all_achalls = [] + for aauthzr in self.aauthzrs: + all_achalls.extend(aauthzr.achalls) + + return all_achalls + def _respond(self, resp, best_effort): """Send/Receive confirmation of all challenges. @@ -139,69 +150,67 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 chall_update = dict() - active_achalls = self._send_responses(self.achalls, - resp, chall_update) + active_achalls = self._send_responses(resp, chall_update) # Check for updated status... try: self._poll_challenges(chall_update, best_effort) finally: - # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) - def _send_responses(self, achalls, resps, chall_update): + def _send_responses(self, resps, chall_update): """Send responses and make sure errors are handled. :param dict chall_update: parameter that is updated to hold - authzr -> list of outstanding solved annotated challenges + aauthzr index to list of outstanding solved annotated challenges """ active_achalls = [] - for achall, resp in six.moves.zip(achalls, resps): - # This line needs to be outside of the if block below to - # ensure failed challenges are cleaned up correctly - active_achalls.append(achall) + resps_iter = iter(resps) + for i, aauthzr in enumerate(self.aauthzrs): + for achall in aauthzr.achalls: + # This line needs to be outside of the if block below to + # ensure failed challenges are cleaned up correctly + active_achalls.append(achall) - # Don't send challenges for None and False authenticator responses - if resp is not None and resp: - self.acme.answer_challenge(achall.challb, resp) - # TODO: answer_challenge returns challr, with URI, - # that can be used in _find_updated_challr - # comparisons... - if achall.domain in chall_update: - chall_update[achall.domain].append(achall) - else: - chall_update[achall.domain] = [achall] + resp = next(resps_iter) + # Don't send challenges for None and False authenticator responses + if resp: + self.acme.answer_challenge(achall.challb, resp) + # TODO: answer_challenge returns challr, with URI, + # that can be used in _find_updated_challr + # comparisons... + chall_update.setdefault(i, []).append(achall) return active_achalls def _poll_challenges( self, chall_update, best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" - dom_to_check = set(chall_update.keys()) - comp_domains = set() + indices_to_check = set(chall_update.keys()) + comp_indices = set() rounds = 0 - while dom_to_check and rounds < max_rounds: + while indices_to_check and rounds < max_rounds: # TODO: Use retry-after... time.sleep(min_sleep) all_failed_achalls = set() - for domain in dom_to_check: + for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( - domain, chall_update[domain]) + index, chall_update[index]) - if len(comp_achalls) == len(chall_update[domain]): - comp_domains.add(domain) + if len(comp_achalls) == len(chall_update[index]): + comp_indices.add(index) elif not failed_achalls: for achall, _ in comp_achalls: - chall_update[domain].remove(achall) + chall_update[index].remove(achall) # We failed some challenges... damage control else: if best_effort: - comp_domains.add(domain) + comp_indices.add(index) logger.warning( "Challenge failed for domain %s", - domain) + self.aauthzrs[index].authzr.body.identifier.value) else: all_failed_achalls.update( updated for _, updated in failed_achalls) @@ -210,24 +219,26 @@ class AuthHandler(object): _report_failed_challs(all_failed_achalls) raise errors.FailedChallenges(all_failed_achalls) - dom_to_check -= comp_domains - comp_domains.clear() + indices_to_check -= comp_indices + comp_indices.clear() rounds += 1 - def _handle_check(self, domain, achalls): + def _handle_check(self, index, achalls): """Returns tuple of ('completed', 'failed').""" completed = [] failed = [] - self.authzr[domain], _ = self.acme.poll(self.authzr[domain]) - if self.authzr[domain].body.status == messages.STATUS_VALID: + original_aauthzr = self.aauthzrs[index] + updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) + self.aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) + if updated_authzr.body.status == messages.STATUS_VALID: return achalls, [] # Note: if the whole authorization is invalid, the individual failed # challenges will be determined here... for achall in achalls: updated_achall = achall.update(challb=self._find_updated_challb( - self.authzr[domain], achall)) + updated_authzr, achall)) # This does nothing for challenges that have yet to be decided yet. if updated_achall.status == messages.STATUS_VALID: @@ -285,14 +296,17 @@ class AuthHandler(object): logger.info("Cleaning up challenges") if achall_list is None: - achalls = self.achalls + achalls = self._get_all_achalls() else: achalls = achall_list if achalls: self.auth.cleanup(achalls) for achall in achalls: - self.achalls.remove(achall) + for aauthzr in self.aauthzrs: + if achall in aauthzr.achalls: + aauthzr.achalls.remove(achall) + break def verify_authzr_complete(self): """Verifies that all authorizations have been decided. @@ -301,15 +315,16 @@ class AuthHandler(object): :rtype: bool """ - for authzr in self.authzr.values(): + for aauthzr in self.aauthzrs: + authzr = aauthzr.authzr if (authzr.body.status != messages.STATUS_VALID and authzr.body.status != messages.STATUS_INVALID): raise errors.AuthorizationError("Incomplete authorizations") - def _challenge_factory(self, domain, path): + def _challenge_factory(self, authzr, path): """Construct Namedtuple Challenges - :param str domain: domain of the enrollee + :param messages.AuthorizationResource authzr: authorization :param list path: List of indices from `challenges`. @@ -323,8 +338,9 @@ class AuthHandler(object): achalls = [] for index in path: - challb = self.authzr[domain].body.challenges[index] - achalls.append(challb_to_achall(challb, self.account.key, domain)) + challb = authzr.body.challenges[index] + achalls.append(challb_to_achall( + challb, self.account.key, authzr.body.identifier.value)) return achalls diff --git a/certbot/cli.py b/certbot/cli.py index 09dd71d13..1c2273c8a 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -599,6 +599,11 @@ class HelpfulArgumentParser(object): if parsed_args.validate_hooks: hooks.validate_hooks(parsed_args) + if parsed_args.allow_subset_of_names: + if any(util.is_wildcard_domain(d) for d in parsed_args.domains): + raise errors.Error("Using --allow-subset-of-names with a" + " wildcard domain is not supported.") + possible_deprecation_warning(parsed_args) return parsed_args diff --git a/certbot/client.py b/certbot/client.py index 0f4fa760d..81fc0b802 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -298,7 +298,12 @@ class Client(object): auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] - if successful_domains != domains: + # allow_subset_of_names is currently disabled for wildcard + # certificates. The reason for this and checking allow_subset_of_names + # below is because successful_domains == domains is never true if + # domains contains a wildcard because the ACME spec forbids identifiers + # in authzs from containing a wildcard character. + if self.config.allow_subset_of_names and successful_domains != domains: if not self.config.dry_run: os.remove(key.file) os.remove(csr.file) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 394002206..7650d2c95 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -29,32 +29,31 @@ class ChallengeFactoryTest(unittest.TestCase): # Account is mocked... self.handler = AuthHandler(None, None, mock.Mock(key="mock_key"), []) - self.dom = "test" - self.handler.authzr[self.dom] = acme_util.gen_authzr( - messages.STATUS_PENDING, self.dom, acme_util.CHALLENGES, + self.authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", acme_util.CHALLENGES, [messages.STATUS_PENDING] * 6, False) def test_all(self): achalls = self.handler._challenge_factory( - self.dom, range(0, len(acme_util.CHALLENGES))) + self.authzr, range(0, len(acme_util.CHALLENGES))) self.assertEqual( [achall.chall for achall in achalls], acme_util.CHALLENGES) def test_one_tls_sni(self): - achalls = self.handler._challenge_factory(self.dom, [1]) + achalls = self.handler._challenge_factory(self.authzr, [1]) self.assertEqual( [achall.chall for achall in achalls], [acme_util.TLSSNI01]) def test_unrecognized(self): - self.handler.authzr["failure.com"] = acme_util.gen_authzr( - messages.STATUS_PENDING, "failure.com", - [mock.Mock(chall="chall", typ="unrecognized")], - [messages.STATUS_PENDING]) + authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", + [mock.Mock(chall="chall", typ="unrecognized")], + [messages.STATUS_PENDING]) self.assertRaises( - errors.Error, self.handler._challenge_factory, "failure.com", [0]) + errors.Error, self.handler._challenge_factory, authzr, [0]) class HandleAuthorizationsTest(unittest.TestCase): @@ -103,7 +102,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -134,7 +133,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -160,7 +159,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -190,12 +189,12 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] self.assertEqual(len(list(six.iterkeys(chall_update))), 3) - self.assertTrue("0" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["0"]), 1) - self.assertTrue("1" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["1"]), 1) - self.assertTrue("2" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["2"]), 1) + self.assertTrue(0 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[0]), 1) + self.assertTrue(1 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[1]), 1) + self.assertTrue(2 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[2]), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -274,14 +273,15 @@ class HandleAuthorizationsTest(unittest.TestCase): self._test_preferred_challenges_not_supported_common(combos=False) def _validate_all(self, unused_1, unused_2): - for dom in six.iterkeys(self.handler.authzr): - azr = self.handler.authzr[dom] - self.handler.authzr[dom] = acme_util.gen_authzr( + for i, aauthzr in enumerate(self.handler.aauthzrs): + azr = aauthzr.authzr + updated_azr = acme_util.gen_authzr( messages.STATUS_VALID, - dom, + azr.body.identifier.value, [challb.chall for challb in azr.body.challenges], [messages.STATUS_VALID] * len(azr.body.challenges), azr.body.combinations) + self.handler.aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) class PollChallengesTest(unittest.TestCase): @@ -290,7 +290,7 @@ class PollChallengesTest(unittest.TestCase): def setUp(self): from certbot.auth_handler import challb_to_achall - from certbot.auth_handler import AuthHandler + from certbot.auth_handler import AuthHandler, AnnotatedAuthzr # Account and network are mocked... self.mock_net = mock.MagicMock() @@ -298,40 +298,38 @@ class PollChallengesTest(unittest.TestCase): None, self.mock_net, mock.Mock(key="mock_key"), []) self.doms = ["0", "1", "2"] - self.handler.authzr[self.doms[0]] = acme_util.gen_authzr( + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[0], [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False) - - self.handler.authzr[self.doms[1]] = acme_util.gen_authzr( + [messages.STATUS_PENDING] * 2, False), [])) + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) - - self.handler.authzr[self.doms[2]] = acme_util.gen_authzr( + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) self.chall_update = {} - for dom in self.doms: - self.chall_update[dom] = [ - challb_to_achall(challb, mock.Mock(key="dummy_key"), dom) - for challb in self.handler.authzr[dom].body.challenges] + for i, aauthzr in enumerate(self.handler.aauthzrs): + self.chall_update[i] = [ + challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) + for challb in aauthzr.authzr.body.challenges] @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid self.handler._poll_challenges(self.chall_update, False) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_VALID) + for aauthzr in self.handler.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) @mock.patch("certbot.auth_handler.time") def test_poll_challenges_failure_best_effort(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid self.handler._poll_challenges(self.chall_update, True) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_PENDING) + for aauthzr in self.handler.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) @mock.patch("certbot.auth_handler.time") @test_util.patch_get_utility() @@ -345,7 +343,7 @@ class PollChallengesTest(unittest.TestCase): def test_unable_to_find_challenge_status(self, unused_mock_time): from certbot.auth_handler import challb_to_achall self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.chall_update[self.doms[0]].append( + self.chall_update[0].append( challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index c5935d722..1bba6991a 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -426,6 +426,10 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods namespace = self.parse(["--no-delete-after-revoke"]) self.assertFalse(namespace.delete_after_revoke) + def test_allow_subset_with_wildcard(self): + self.assertRaises(errors.Error, self.parse, + "--allow-subset-of-names -d *.example.org".split()) + class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index ed9c140e7..b51275d9e 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -222,6 +222,7 @@ class ClientTest(ClientTestCommon): mock.sentinel.chain) authzr = self._authzr_from_domains(["example.com"]) + self.config.allow_subset_of_names = True self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 50d323ffd..0e280f3ab 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -487,6 +487,26 @@ class EnforceDomainSanityTest(unittest.TestCase): self._call('this.is.xn--ls8h.tld') +class IsWildcardDomainTest(unittest.TestCase): + """Tests for is_wildcard_domain.""" + + def setUp(self): + self.wildcard = u"*.example.org" + self.no_wildcard = u"example.org" + + def _call(self, domain): + from certbot.util import is_wildcard_domain + return is_wildcard_domain(domain) + + def test_no_wildcard(self): + self.assertFalse(self._call(self.no_wildcard)) + self.assertFalse(self._call(self.no_wildcard.encode())) + + def test_wildcard(self): + self.assertTrue(self._call(self.wildcard)) + self.assertTrue(self._call(self.wildcard.encode())) + + class OsInfoTest(unittest.TestCase): """Test OS / distribution detection""" diff --git a/certbot/util.py b/certbot/util.py index f47c5da9c..f7ce6a3bc 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -601,6 +601,24 @@ def enforce_domain_sanity(domain): return domain +def is_wildcard_domain(domain): + """"Is domain a wildcard domain? + + :param damain: domain to check + :type domain: `bytes` or `str` or `unicode` + + :returns: True if domain is a wildcard, otherwise, False + :rtype: bool + + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + + return domain.startswith(wildcard_marker) + + def get_strict_version(normalized): """Converts a normalized version to a strict version. From e9bc4a319b9989a300dd574466a15edd581ee3c4 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 28 Feb 2018 21:31:47 +0200 Subject: [PATCH 73/93] Apache plugin wildcard support for ACMEv2 (#5608) In `deploy_cert()` and `enhance()`, the user will be presented with a dialog to choose from the VirtualHosts that can be covered by the wildcard domain name. The (multiple) selection result will then be handled in a similar way that we previously handled a single VirtualHost that was returned by the `_find_best_vhost()`. Additionally the selected VirtualHosts are added to a dictionary that maps selections to a wildcard domain to be reused in the later `enhance()` call and not forcing the user to select the same VirtualHosts again. * Apache plugin wildcard support * Present dialog only once per domain, added tests * Raise exception if no VHosts selected for wildcard domain --- certbot-apache/certbot_apache/configurator.py | 141 +++++++++++++++++- certbot-apache/certbot_apache/display_ops.py | 48 +++++- certbot-apache/certbot_apache/obj.py | 13 ++ .../certbot_apache/tests/configurator_test.py | 100 +++++++++++++ .../certbot_apache/tests/display_ops_test.py | 30 ++++ certbot/tests/main_test.py | 1 - 6 files changed, 316 insertions(+), 17 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 4bb2cbebd..6377bb114 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -5,6 +5,7 @@ import logging import os import pkg_resources import re +import six import socket import time @@ -152,6 +153,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc = dict() # Outstanding challenges self._chall_out = set() + # List of vhosts configured per wildcard domain on this run. + # used by deploy_cert() and enhance() + self._wildcard_vhosts = dict() # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) @@ -262,6 +266,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.aug, self.conf("server-root"), self.conf("vhost-root"), self.version, configurator=self) + def _wildcard_domain(self, domain): + """ + Checks if domain is a wildcard domain + + :param str domain: Domain to check + + :returns: If the domain is wildcard domain + :rtype: bool + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + return domain.startswith(wildcard_marker) + def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): """Deploys certificate to specified virtual host. @@ -280,9 +299,112 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): a lack of directives """ - # Choose vhost before (possible) enabling of mod_ssl, to keep the - # vhost choice namespace similar with the pre-validation one. - vhost = self.choose_vhost(domain) + vhosts = self.choose_vhosts(domain) + for vhost in vhosts: + self._deploy_cert(vhost, cert_path, key_path, chain_path, fullchain_path) + + def choose_vhosts(self, domain, create_if_no_ssl=True): + """ + Finds VirtualHosts that can be used with the provided domain + + :param str domain: Domain name to match VirtualHosts to + :param bool create_if_no_ssl: If found VirtualHost doesn't have a HTTPS + counterpart, should one get created + + :returns: List of VirtualHosts or None + :rtype: `list` of :class:`~certbot_apache.obj.VirtualHost` + """ + + if self._wildcard_domain(domain): + if domain in self._wildcard_vhosts: + # Vhosts for a wildcard domain were already selected + return self._wildcard_vhosts[domain] + # Ask user which VHosts to support. + # Returned objects are guaranteed to be ssl vhosts + return self._choose_vhosts_wildcard(domain, create_if_no_ssl) + else: + return [self.choose_vhost(domain)] + + def _vhosts_for_wildcard(self, domain): + """ + Get VHost objects for every VirtualHost that the user wants to handle + with the wildcard certificate. + """ + + # Collect all vhosts that match the name + matched = set() + for vhost in self.vhosts: + for name in vhost.get_names(): + if self._in_wildcard_scope(name, domain): + matched.add(vhost) + + return list(matched) + + def _in_wildcard_scope(self, name, domain): + """ + Helper method for _vhosts_for_wildcard() that makes sure that the domain + is in the scope of wildcard domain. + + eg. in scope: domain = *.wild.card, name = 1.wild.card + not in scope: domain = *.wild.card, name = 1.2.wild.card + """ + if len(name.split(".")) == len(domain.split(".")): + return fnmatch.fnmatch(name, domain) + + + def _choose_vhosts_wildcard(self, domain, create_ssl=True): + """Prompts user to choose vhosts to install a wildcard certificate for""" + + # Get all vhosts that are covered by the wildcard domain + vhosts = self._vhosts_for_wildcard(domain) + + # Go through the vhosts, making sure that we cover all the names + # present, but preferring the SSL vhosts + filtered_vhosts = dict() + for vhost in vhosts: + for name in vhost.get_names(): + if vhost.ssl: + # Always prefer SSL vhosts + filtered_vhosts[name] = vhost + elif name not in filtered_vhosts and create_ssl: + # Add if not in list previously + filtered_vhosts[name] = vhost + + # Only unique VHost objects + dialog_input = set([vhost for vhost in filtered_vhosts.values()]) + + # Ask the user which of names to enable, expect list of names back + dialog_output = display_ops.select_vhost_multiple(list(dialog_input)) + + if not dialog_output: + logger.error( + "No vhost exists with servername or alias for domain %s. " + "No vhost was selected. Please specify ServerName or ServerAlias " + "in the Apache config.", + domain) + raise errors.PluginError("No vhost selected") + + # Make sure we create SSL vhosts for the ones that are HTTP only + # if requested. + return_vhosts = list() + for vhost in dialog_output: + if not vhost.ssl: + return_vhosts.append(self.make_vhost_ssl(vhost)) + else: + return_vhosts.append(vhost) + + self._wildcard_vhosts[domain] = return_vhosts + return return_vhosts + + + def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): + """ + Helper function for deploy_cert() that handles the actual deployment + this exists because we might want to do multiple deployments per + domain originally passed for deploy_cert(). This is especially true + with wildcard certificates + """ + # This is done first so that ssl module is enabled and cert_path, # cert_key... can all be parsed appropriately @@ -311,7 +433,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "Unable to find cert and/or key directives") - logger.info("Deploying Certificate for %s to VirtualHost %s", domain, vhost.filep) + logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) if self.version < (2, 4, 8) or (chain_path and not fullchain_path): # install SSLCertificateFile, SSLCertificateKeyFile, @@ -327,8 +449,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "version of Apache") else: if not fullchain_path: - raise errors.PluginError("Please provide the --fullchain-path\ - option pointing to your full chain file") + raise errors.PluginError("Please provide the --fullchain-path " + "option pointing to your full chain file") set_cert_path = fullchain_path self.aug.set(path["cert_path"][-1], fullchain_path) self.aug.set(path["cert_key"][-1], key_path) @@ -391,7 +513,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.error( "No vhost exists with servername or alias of %s. " "No vhost was selected. Please specify ServerName or ServerAlias " - "in the Apache config, or split vhosts into separate files.", + "in the Apache config.", target_name) raise errors.PluginError("No vhost selected") elif temp: @@ -1376,8 +1498,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): except KeyError: raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) + + vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) try: - func(self.choose_vhost(domain), options) + for vhost in vhosts: + func(vhost, options) except errors.PluginError: logger.warning("Failed %s for %s", enhancement, domain) raise diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index 9529c1ab3..097b84b96 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -13,10 +13,44 @@ import certbot.display.util as display_util logger = logging.getLogger(__name__) +def select_vhost_multiple(vhosts): + """Select multiple Vhosts to install the certificate for + + :param vhosts: Available Apache VirtualHosts + :type vhosts: :class:`list` of type `~obj.Vhost` + + :returns: List of VirtualHosts + :rtype: :class:`list`of type `~obj.Vhost` + """ + if not vhosts: + return list() + tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] + # Remove the extra newline from the last entry + if len(tags_list): + tags_list[-1] = tags_list[-1][:-1] + code, names = zope.component.getUtility(interfaces.IDisplay).checklist( + "Which VirtualHosts would you like to install the wildcard certificate for?", + tags=tags_list, force_interactive=True) + if code == display_util.OK: + return_vhosts = _reversemap_vhosts(names, vhosts) + return return_vhosts + return [] + +def _reversemap_vhosts(names, vhosts): + """Helper function for select_vhost_multiple for mapping string + representations back to actual vhost objects""" + return_vhosts = list() + + for selection in names: + for vhost in vhosts: + if vhost.display_repr().strip() == selection.strip(): + return_vhosts.append(vhost) + return return_vhosts + def select_vhost(domain, vhosts): """Select an appropriate Apache Vhost. - :param vhosts: Available Apache Virtual Hosts + :param vhosts: Available Apache VirtualHosts :type vhosts: :class:`list` of type `~obj.Vhost` :returns: VirtualHost or `None` @@ -25,13 +59,11 @@ def select_vhost(domain, vhosts): """ if not vhosts: return None - while True: - code, tag = _vhost_menu(domain, vhosts) - if code == display_util.OK: - return vhosts[tag] - else: - return None - + code, tag = _vhost_menu(domain, vhosts) + if code == display_util.OK: + return vhosts[tag] + else: + return None def _vhost_menu(domain, vhosts): """Select an appropriate Apache Vhost. diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index 1e3579858..fcf3bfe08 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -167,6 +167,19 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods active="Yes" if self.enabled else "No", modmacro="Yes" if self.modmacro else "No")) + def display_repr(self): + """Return a representation of VHost to be used in dialog""" + return ( + "File: {filename}\n" + "Addresses: {addrs}\n" + "Names: {names}\n" + "HTTPS: {https}\n".format( + filename=self.filep, + addrs=", ".join(str(addr) for addr in self.addrs), + names=", ".join(self.get_names()), + https="Yes" if self.ssl else "No")) + + def __eq__(self, other): if isinstance(other, self.__class__): return (self.filep == other.filep and self.path == other.path and diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 8f34d33d3..c9bf9a63f 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -1337,6 +1337,106 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enable_mod, "whatever") + def test_wildcard_domain(self): + # pylint: disable=protected-access + cases = {u"*.example.org": True, b"*.x.example.org": True, + u"a.example.org": False, b"a.x.example.org": False} + for key in cases.keys(): + self.assertEqual(self.config._wildcard_domain(key), cases[key]) + + def test_choose_vhosts_wildcard(self): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[3]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=True) + # Check that the dialog was called with one vh: certbot.demo + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[3]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].name == "certbot.demo") + self.assertTrue(vhs[0].ssl) + + self.assertFalse(vhs[0] == self.vh_truth[3]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_no_ssl(self, mock_makessl): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[1]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=False) + self.assertFalse(mock_makessl.called) + self.assertEquals(vhs[0], self.vh_truth[1]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._vhosts_for_wildcard") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_already_ssl(self, mock_makessl, mock_vh_for_w): + # pylint: disable=protected-access + # Already SSL vhost + mock_vh_for_w.return_value = [self.vh_truth[7]] + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[7]] + vhs = self.config._choose_vhosts_wildcard("whatever", + create_ssl=True) + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[7]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + # Ensure that make_vhost_ssl was not called, vhost.ssl == true + self.assertFalse(mock_makessl.called) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].ssl) + self.assertEquals(vhs[0], self.vh_truth[7]) + + + def test_deploy_cert_wildcard(self): + # pylint: disable=protected-access + mock_choose_vhosts = mock.MagicMock() + mock_choose_vhosts.return_value = [self.vh_truth[7]] + self.config._choose_vhosts_wildcard = mock_choose_vhosts + mock_d = "certbot_apache.configurator.ApacheConfigurator._deploy_cert" + with mock.patch(mock_d) as mock_dep: + self.config.deploy_cert("*.wildcard.example.org", "/tmp/path", + "/tmp/path", "/tmp/path", "/tmp/path") + self.assertTrue(mock_dep.called) + self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(self.vh_truth[7], mock_dep.call_args_list[0][0][0]) + + @mock.patch("certbot_apache.display_ops.select_vhost_multiple") + def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog): + # pylint: disable=protected-access + mock_dialog.return_value = [] + self.assertRaises(errors.PluginError, + self.config.deploy_cert, + "*.wild.cat", "/tmp/path", "/tmp/path", + "/tmp/path", "/tmp/path") + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_after_install(self, mock_choose): + # pylint: disable=protected-access + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertFalse(mock_choose.called) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_no_install(self, mock_choose): + mock_choose.return_value = [self.vh_truth[3]] + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertTrue(mock_choose.called) + + class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" # pylint: disable=protected-access diff --git a/certbot-apache/certbot_apache/tests/display_ops_test.py b/certbot-apache/certbot_apache/tests/display_ops_test.py index e59d411bd..df5cdbac0 100644 --- a/certbot-apache/certbot_apache/tests/display_ops_test.py +++ b/certbot-apache/certbot_apache/tests/display_ops_test.py @@ -11,9 +11,39 @@ from certbot.tests import util as certbot_util from certbot_apache import obj +from certbot_apache.display_ops import select_vhost_multiple from certbot_apache.tests import util +class SelectVhostMultiTest(unittest.TestCase): + """Tests for certbot_apache.display_ops.select_vhost_multiple.""" + + def setUp(self): + self.base_dir = "/example_path" + self.vhosts = util.get_vh_truth( + self.base_dir, "debian_apache_2_4/multiple_vhosts") + + def test_select_no_input(self): + self.assertFalse(select_vhost_multiple([])) + + @certbot_util.patch_get_utility() + def test_select_correct(self, mock_util): + mock_util().checklist.return_value = ( + display_util.OK, [self.vhosts[3].display_repr(), + self.vhosts[2].display_repr()]) + vhs = select_vhost_multiple([self.vhosts[3], + self.vhosts[2], + self.vhosts[1]]) + self.assertTrue(self.vhosts[2] in vhs) + self.assertTrue(self.vhosts[3] in vhs) + self.assertFalse(self.vhosts[1] in vhs) + + @certbot_util.patch_get_utility() + def test_select_cancel(self, mock_util): + mock_util().checklist.return_value = (display_util.CANCEL, "whatever") + vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]]) + self.assertFalse(vhs) + class SelectVhostTest(unittest.TestCase): """Tests for certbot_apache.display_ops.select_vhost.""" diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index c31a3fb33..b778f05ea 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -940,7 +940,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertRaises(errors.ConfigurationError, self._call, ['-d', (('a' * 50) + '.') * 10]) - # Bare IP address (this is actually a different error message now) self.assertRaises(errors.ConfigurationError, self._call, From 78735fa2c3ebaee6c7aba02bbf939597a9075cbb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 28 Feb 2018 16:08:06 -0800 Subject: [PATCH 74/93] Suggest DNS authenticator when it's needed (#5638) --- certbot/auth_handler.py | 16 ++++++++++++---- certbot/tests/auth_handler_test.py | 6 ++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 2b38e4af5..67d36c8cc 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -433,7 +433,7 @@ def _find_smart_path(challbs, preferences, combinations): combo_total = 0 if not best_combo: - _report_no_chall_path() + _report_no_chall_path(challbs) return best_combo @@ -454,15 +454,23 @@ def _find_dumb_path(challbs, preferences): if supported: path.append(i) else: - _report_no_chall_path() + _report_no_chall_path(challbs) return path -def _report_no_chall_path(): - """Logs and raises an error that no satisfiable chall path exists.""" +def _report_no_chall_path(challbs): + """Logs and raises an error that no satisfiable chall path exists. + + :param challbs: challenges from the authorization that can't be satisfied + + """ msg = ("Client with the currently selected authenticator does not support " "any combination of challenges that will satisfy the CA.") + if len(challbs) == 1 and isinstance(challbs[0].chall, challenges.DNS01): + msg += ( + " You may need to use an authenticator " + "plugin that can do challenges over DNS.") logger.fatal(msg) raise errors.AuthorizationError(msg) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 7650d2c95..b6af3d0f5 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -272,6 +272,12 @@ class HandleAuthorizationsTest(unittest.TestCase): self.mock_net.acme_version = 2 self._test_preferred_challenges_not_supported_common(combos=False) + def test_dns_only_challenge_not_supported(self): + authzrs = [gen_dom_authzr(domain="0", challs=[acme_util.DNS01])] + mock_order = mock.MagicMock(authorizations=authzrs) + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def _validate_all(self, unused_1, unused_2): for i, aauthzr in enumerate(self.handler.aauthzrs): azr = aauthzr.authzr From 38d5144fff79e10507ee7d53cb6664c8180d2245 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 08:25:32 -0800 Subject: [PATCH 75/93] Drop min coverage to 63 (#5641) --- tests/boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 24d224cb0..ea412b6b9 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -437,4 +437,4 @@ then . ./certbot-nginx/tests/boulder-integration.sh fi -coverage report --fail-under 64 -m +coverage report --fail-under 63 -m From 559220c2eff90975fb671945d780c2a757d0e167 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 10:11:15 -0800 Subject: [PATCH 76/93] Add basic ACMEv2 integration tests (#5635) * Use newer boulder config * Use ACMEv2 endpoint if requested * Add v2 integration tests * Work with unset variables * Add wildcard issuance test * quote domains --- .travis.yml | 6 +++++- tests/boulder-fetch.sh | 7 +++++++ tests/boulder-integration.sh | 6 ++++++ tests/integration/_common.sh | 5 +++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 42b8d679d..c62664180 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,11 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=1 + env: TOXENV=py27_install BOULDER_INTEGRATION=v1 + sudo: required + services: docker + - python: "2.7" + env: TOXENV=py27_install BOULDER_INTEGRATION=v2 sudo: required services: docker - python: "2.7" diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 08eb736c2..fc9cbaae7 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -16,6 +16,13 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml + +# If we're testing against ACMEv2, we need to use a newer boulder config for +# now. See https://github.com/letsencrypt/boulder#quickstart. +if [ "$BOULDER_INTEGRATION" = "v2" ]; then + sed -i 's/BOULDER_CONFIG_DIR: .*/BOULDER_CONFIG_DIR: test\/config-next/' docker-compose.yml +fi + docker-compose up -d set +x # reduce verbosity while waiting for boulder diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ea412b6b9..f2b0dcf60 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -430,6 +430,12 @@ for path in $archive $conf $live; do fi done +# Test ACMEv2-only features +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then + common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ + --manual-auth-hook ./tests/manual-dns-auth.sh +fi + # Most CI systems set this variable to true. # If the tests are running as part of CI, Nginx should be available. if ${CI:-false} || type nginx; diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index d151bdc3f..236090a14 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,6 +16,11 @@ certbot_test () { "$@" } +# Use local ACMEv2 endpoint if requested and SERVER isn't already set. +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" -a -z "${SERVER:+x}" ]; then + SERVER="http://localhost:4001/directory" +fi + certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" omit_patterns="$omit_patterns,*_test.py,*_test_*," From f0b337532cdc232add47c7bf98401eb7d75ca615 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 1 Mar 2018 14:05:50 -0800 Subject: [PATCH 77/93] Nginx plugin wildcard support for ACMEv2 (#5619) * support wildcards for deploy_cert * support wildcards for enhance * redirect enhance and some tests * update tests * add display_ops and display_repr * update display_ops_test and errors found * say server block * match redirects properly * functional code * start adding tests and lint errors * add configurator tests * lint * change message to be generic to installation and enhancement * remove _wildcard_domain * take selecting vhosts out of loop * remove extra newline * filter wildcard vhosts by port * lint * don't filter by domain * [^.]+ * lint * make vhost hashable * one more tuple --- certbot-nginx/certbot_nginx/configurator.py | 199 ++++++++++++++---- certbot-nginx/certbot_nginx/display_ops.py | 44 ++++ certbot-nginx/certbot_nginx/http_01.py | 6 +- certbot-nginx/certbot_nginx/obj.py | 17 ++ .../certbot_nginx/tests/configurator_test.py | 100 ++++++++- .../certbot_nginx/tests/display_ops_test.py | 45 ++++ .../certbot_nginx/tests/tls_sni_01_test.py | 4 +- certbot-nginx/certbot_nginx/tls_sni_01.py | 7 +- 8 files changed, 370 insertions(+), 52 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/display_ops.py create mode 100644 certbot-nginx/certbot_nginx/tests/display_ops_test.py diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 9f091c0fd..e4d87744e 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -23,6 +23,7 @@ from certbot import util from certbot.plugins import common from certbot_nginx import constants +from certbot_nginx import display_ops from certbot_nginx import nginxparser from certbot_nginx import parser from certbot_nginx import tls_sni_01 @@ -92,6 +93,11 @@ class NginxConfigurator(common.Installer): # For creating new vhosts if no names match self.new_vhost = None + # List of vhosts configured per wildcard domain on this run. + # used by deploy_cert() and enhance() + self._wildcard_vhosts = {} + self._wildcard_redirect_vhosts = {} + # Add number of outstanding challenges self._chall_out = 0 @@ -146,6 +152,7 @@ class NginxConfigurator(common.Installer): raise errors.PluginError( 'Unable to lock %s', self.conf('server-root')) + # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): @@ -166,14 +173,24 @@ class NginxConfigurator(common.Installer): "The nginx plugin currently requires --fullchain-path to " "install a cert.") - vhost = self.choose_vhost(domain, create_if_no_match=True) + vhosts = self.choose_vhosts(domain, create_if_no_match=True) + for vhost in vhosts: + self._deploy_cert(vhost, cert_path, key_path, chain_path, fullchain_path) + + def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): + # pylint: disable=unused-argument + """ + Helper function for deploy_cert() that handles the actual deployment + this exists because we might want to do multiple deployments per + domain originally passed for deploy_cert(). This is especially true + with wildcard certificates + """ cert_directives = [['\n ', 'ssl_certificate', ' ', fullchain_path], ['\n ', 'ssl_certificate_key', ' ', key_path]] self.parser.add_server_directives(vhost, cert_directives, replace=True) - logger.info("Deployed Certificate to VirtualHost %s for %s", - vhost.filep, ", ".join(vhost.names)) + logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, @@ -181,10 +198,61 @@ class NginxConfigurator(common.Installer): self.save_notes += "\tssl_certificate %s\n" % fullchain_path self.save_notes += "\tssl_certificate_key %s\n" % key_path + def _choose_vhosts_wildcard(self, domain, prefer_ssl, no_ssl_filter_port=None): + """Prompts user to choose vhosts to install a wildcard certificate for""" + if prefer_ssl: + vhosts_cache = self._wildcard_vhosts + preference_test = lambda x: x.ssl + else: + vhosts_cache = self._wildcard_redirect_vhosts + preference_test = lambda x: not x.ssl + + # Caching! + if domain in vhosts_cache: + # Vhosts for a wildcard domain were already selected + return vhosts_cache[domain] + + # Get all vhosts whether or not they are covered by the wildcard domain + vhosts = self.parser.get_vhosts() + + # Go through the vhosts, making sure that we cover all the names + # present, but preferring the SSL or non-SSL vhosts + filtered_vhosts = {} + for vhost in vhosts: + # Ensure we're listening non-sslishly on no_ssl_filter_port + if no_ssl_filter_port is not None: + if not self._vhost_listening_on_port_no_ssl(vhost, no_ssl_filter_port): + continue + for name in vhost.names: + if preference_test(vhost): + # Prefer either SSL or non-SSL vhosts + filtered_vhosts[name] = vhost + elif name not in filtered_vhosts: + # Add if not in list previously + filtered_vhosts[name] = vhost + + # Only unique VHost objects + dialog_input = set([vhost for vhost in filtered_vhosts.values()]) + + # Ask the user which of names to enable, expect list of names back + return_vhosts = display_ops.select_vhost_multiple(list(dialog_input)) + + for vhost in return_vhosts: + if domain not in vhosts_cache: + vhosts_cache[domain] = [] + vhosts_cache[domain].append(vhost) + + return return_vhosts + ####################### # Vhost parsing methods ####################### - def choose_vhost(self, target_name, create_if_no_match=False): + def _choose_vhost_single(self, target_name): + matches = self._get_ranked_matches(target_name) + vhosts = [x for x in [self._select_best_name_match(matches)] if x is not None] + return vhosts + + def choose_vhosts(self, target_name, create_if_no_match=False): """Chooses a virtual host based on the given domain name. .. note:: This makes the vhost SSL-enabled if it isn't already. Follows @@ -202,17 +270,19 @@ class NginxConfigurator(common.Installer): when there is no match found. If we can't choose a default, raise a MisconfigurationError. - :returns: ssl vhost associated with name - :rtype: :class:`~certbot_nginx.obj.VirtualHost` + :returns: ssl vhosts associated with name + :rtype: list of :class:`~certbot_nginx.obj.VirtualHost` """ - vhost = None - - matches = self._get_ranked_matches(target_name) - vhost = self._select_best_name_match(matches) - if not vhost: + if util.is_wildcard_domain(target_name): + # Ask user which VHosts to support. + vhosts = self._choose_vhosts_wildcard(target_name, prefer_ssl=True) + else: + vhosts = self._choose_vhost_single(target_name) + if not vhosts: if create_if_no_match: - vhost = self._vhost_from_duplicated_default(target_name) + # result will not be [None] because it errors on failure + vhosts = [self._vhost_from_duplicated_default(target_name)] else: # No matches. Raise a misconfiguration error. raise errors.MisconfigurationError( @@ -222,10 +292,11 @@ class NginxConfigurator(common.Installer): "nginx configuration: " "https://nginx.org/en/docs/http/server_names.html") % (target_name)) # Note: if we are enhancing with ocsp, vhost should already be ssl. - if not vhost.ssl: - self._make_server_ssl(vhost) + for vhost in vhosts: + if not vhost.ssl: + self._make_server_ssl(vhost) - return vhost + return vhosts def ipv6_info(self, port): """Returns tuple of booleans (ipv6_active, ipv6only_present) @@ -359,7 +430,7 @@ class NginxConfigurator(common.Installer): return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhost(self, target_name, port, create_if_no_match=False): + def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -377,15 +448,20 @@ class NginxConfigurator(common.Installer): when there is no match found. If we can't choose a default, raise a MisconfigurationError. - :returns: vhost associated with name - :rtype: :class:`~certbot_nginx.obj.VirtualHost` + :returns: vhosts associated with name + :rtype: list of :class:`~certbot_nginx.obj.VirtualHost` """ - matches = self._get_redirect_ranked_matches(target_name, port) - vhost = self._select_best_name_match(matches) - if not vhost and create_if_no_match: - vhost = self._vhost_from_duplicated_default(target_name, port=port) - return vhost + if util.is_wildcard_domain(target_name): + # Ask user which VHosts to enhance. + vhosts = self._choose_vhosts_wildcard(target_name, prefer_ssl=False, + no_ssl_filter_port=port) + else: + matches = self._get_redirect_ranked_matches(target_name, port) + vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] + if not vhosts and create_if_no_match: + vhosts = [self._vhost_from_duplicated_default(target_name, port=port)] + return vhosts def _port_matches(self, test_port, matching_port): # test_port is a number, matching is a number or "" or None @@ -395,6 +471,23 @@ class NginxConfigurator(common.Installer): else: return test_port == matching_port + def _vhost_listening_on_port_no_ssl(self, vhost, port): + found_matching_port = False + if len(vhost.addrs) == 0: + # if there are no listen directives at all, Nginx defaults to + # listening on port 80. + found_matching_port = (port == self.DEFAULT_LISTEN_PORT) + else: + for addr in vhost.addrs: + if self._port_matches(port, addr.get_port()) and addr.ssl == False: + found_matching_port = True + + if found_matching_port: + # make sure we don't have an 'ssl on' directive + return not self.parser.has_ssl_on_directive(vhost) + else: + return False + def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -411,21 +504,7 @@ class NginxConfigurator(common.Installer): all_vhosts = self.parser.get_vhosts() def _vhost_matches(vhost, port): - found_matching_port = False - if len(vhost.addrs) == 0: - # if there are no listen directives at all, Nginx defaults to - # listening on port 80. - found_matching_port = (port == self.DEFAULT_LISTEN_PORT) - else: - for addr in vhost.addrs: - if self._port_matches(port, addr.get_port()) and addr.ssl == False: - found_matching_port = True - - if found_matching_port: - # make sure we don't have an 'ssl on' directive - return not self.parser.has_ssl_on_directive(vhost) - else: - return False + return self._vhost_listening_on_port_no_ssl(vhost, port) matching_vhosts = [vhost for vhost in all_vhosts if _vhost_matches(vhost, port)] @@ -587,17 +666,31 @@ class NginxConfigurator(common.Installer): """ port = self.DEFAULT_LISTEN_PORT - vhost = None # If there are blocks listening plaintextishly on self.DEFAULT_LISTEN_PORT, # choose the most name-matching one. - vhost = self.choose_redirect_vhost(domain, port) + vhosts = self.choose_redirect_vhosts(domain, port) - if vhost is None: + if not vhosts: logger.info("No matching insecure server blocks listening on port %s found.", self.DEFAULT_LISTEN_PORT) return + for vhost in vhosts: + self._enable_redirect_single(domain, vhost) + + def _enable_redirect_single(self, domain, vhost): + """Redirect all equivalent HTTP traffic to ssl_vhost. + + If the vhost is listening plaintextishly, separate out the + relevant directives into a new server block and add a rewrite directive. + + .. note:: This function saves the configuration + + :param str domain: domain to enable redirect for + :param `~obj.Vhost` vhost: vhost to enable redirect for + """ + new_vhost = None if vhost.ssl: new_vhost = self.parser.duplicate_vhost(vhost, @@ -638,7 +731,18 @@ class NginxConfigurator(common.Installer): :type chain_path: `str` or `None` """ - vhost = self.choose_vhost(domain) + vhosts = self.choose_vhosts(domain) + for vhost in vhosts: + self._enable_ocsp_stapling_single(vhost, chain_path) + + def _enable_ocsp_stapling_single(self, vhost, chain_path): + """Include OCSP response in TLS handshake + + :param str vhost: vhost to enable OCSP response for + :param chain_path: chain file path + :type chain_path: `str` or `None` + + """ if self.version < (1, 3, 7): raise errors.PluginError("Version 1.3.7 or greater of nginx " "is needed to enable OCSP stapling") @@ -889,14 +993,23 @@ def _test_block_from_block(block): parser.comment_directive(test_block, 0) return test_block[:-1] + def _redirect_block_for_domain(domain): + updated_domain = domain + match_symbol = '=' + if util.is_wildcard_domain(domain): + match_symbol = '~' + updated_domain = updated_domain.replace('.', r'\.') + updated_domain = updated_domain.replace('*', '[^.]+') + updated_domain = '^' + updated_domain + '$' redirect_block = [[ - ['\n ', 'if', ' ', '($host', ' ', '=', ' ', '%s)' % domain, ' '], + ['\n ', 'if', ' ', '($host', ' ', match_symbol, ' ', '%s)' % updated_domain, ' '], [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], '\n ']], ['\n']] return redirect_block + def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. diff --git a/certbot-nginx/certbot_nginx/display_ops.py b/certbot-nginx/certbot_nginx/display_ops.py new file mode 100644 index 000000000..5d6bda6b0 --- /dev/null +++ b/certbot-nginx/certbot_nginx/display_ops.py @@ -0,0 +1,44 @@ +"""Contains UI methods for Nginx operations.""" +import logging + +import zope.component + +from certbot import interfaces + +import certbot.display.util as display_util + + +logger = logging.getLogger(__name__) + + +def select_vhost_multiple(vhosts): + """Select multiple Vhosts to install the certificate for + :param vhosts: Available Nginx VirtualHosts + :type vhosts: :class:`list` of type `~obj.Vhost` + :returns: List of VirtualHosts + :rtype: :class:`list`of type `~obj.Vhost` + """ + if not vhosts: + return list() + tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] + # Remove the extra newline from the last entry + if len(tags_list): + tags_list[-1] = tags_list[-1][:-1] + code, names = zope.component.getUtility(interfaces.IDisplay).checklist( + "Which server blocks would you like to modify?", + tags=tags_list, force_interactive=True) + if code == display_util.OK: + return_vhosts = _reversemap_vhosts(names, vhosts) + return return_vhosts + return [] + +def _reversemap_vhosts(names, vhosts): + """Helper function for select_vhost_multiple for mapping string + representations back to actual vhost objects""" + return_vhosts = list() + + for selection in names: + for vhost in vhosts: + if vhost.display_repr().strip() == selection.strip(): + return_vhosts.append(vhost) + return return_vhosts diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index c0dec061a..0b1b2bfe0 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -179,13 +179,17 @@ class NginxHttp01(common.ChallengePerformer): """ try: - vhost = self.configurator.choose_redirect_vhost(achall.domain, + vhosts = self.configurator.choose_redirect_vhosts(achall.domain, '%i' % self.configurator.config.http01_port, create_if_no_match=True) except errors.MisconfigurationError: # Couldn't find either a matching name+port server block # or a port+default_server block, so create a dummy block return self._make_server_block(achall) + # len is max 1 because Nginx doesn't authenticate wildcards + # if len were or vhosts None, we would have errored + vhost = vhosts[0] + # Modify existing server block validation = achall.validation(achall.account_key) validation_path = self._get_validation_path(achall) diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index e8dc8936d..3625a95b9 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -193,6 +193,11 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return False + def __hash__(self): + return hash((self.filep, tuple(self.path), + tuple(self.addrs), tuple(self.names), + self.ssl, self.enabled)) + def contains_list(self, test): """Determine if raw server block contains test list at top level """ @@ -216,3 +221,15 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods for a in self.addrs: if not a.ipv6: return True + + def display_repr(self): + """Return a representation of VHost to be used in dialog""" + return ( + "File: {filename}\n" + "Addresses: {addrs}\n" + "Names: {names}\n" + "HTTPS: {https}\n".format( + filename=self.filep, + addrs=", ".join(str(addr) for addr in self.addrs), + names=", ".join(self.names), + https="Yes" if self.ssl else "No")) diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index acb7ee282..722ba68bf 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -128,7 +128,7 @@ class NginxConfiguratorTest(util.NginxTest): ['#', parser.COMMENT]]]], parsed[0]) - def test_choose_vhost(self): + def test_choose_vhosts(self): localhost_conf = set(['localhost', r'~^(www\.)?(example|bar)\.']) server_conf = set(['somename', 'another.alias', 'alias']) example_conf = set(['.example.com', 'example.*']) @@ -159,7 +159,7 @@ class NginxConfiguratorTest(util.NginxTest): '69.255.225.155'] for name in results: - vhost = self.config.choose_vhost(name) + vhost = self.config.choose_vhosts(name)[0] path = os.path.relpath(vhost.filep, self.temp_dir) self.assertEqual(results[name], vhost.names) @@ -173,7 +173,7 @@ class NginxConfiguratorTest(util.NginxTest): for name in bad_results: self.assertRaises(errors.MisconfigurationError, - self.config.choose_vhost, name) + self.config.choose_vhosts, name) def test_ipv6only(self): # ipv6_info: (ipv6_active, ipv6only_present) @@ -702,6 +702,100 @@ class NginxConfiguratorTest(util.NginxTest): self.config.rollback_checkpoints() self.assertTrue(mock_parser_load.call_count == 3) + def test_choose_vhosts_wildcard(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_select_vhs.return_value = [vhost] + vhs = self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=True) + # Check that the dialog was called with migration.com + self.assertTrue(vhost in mock_select_vhs.call_args[0][0]) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertEqual(vhs[0], vhost) + + def test_choose_vhosts_wildcard_redirect(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_select_vhs.return_value = [vhost] + vhs = self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=False) + # Check that the dialog was called with migration.com + self.assertTrue(vhost in mock_select_vhs.call_args[0][0]) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertEqual(vhs[0], vhost) + + def test_deploy_cert_wildcard(self): + # pylint: disable=protected-access + mock_choose_vhosts = mock.MagicMock() + vhost = [x for x in self.config.parser.get_vhosts() + if 'geese.com' in x.names][0] + mock_choose_vhosts.return_value = [vhost] + self.config._choose_vhosts_wildcard = mock_choose_vhosts + mock_d = "certbot_nginx.configurator.NginxConfigurator._deploy_cert" + with mock.patch(mock_d) as mock_dep: + self.config.deploy_cert("*.com", "/tmp/path", + "/tmp/path", "/tmp/path", "/tmp/path") + self.assertTrue(mock_dep.called) + self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(vhost, mock_dep.call_args_list[0][0][0]) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog): + # pylint: disable=protected-access + mock_dialog.return_value = [] + self.assertRaises(errors.PluginError, + self.config.deploy_cert, + "*.wild.cat", "/tmp/path", "/tmp/path", + "/tmp/path", "/tmp/path") + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_ocsp_after_install(self, mock_dialog): + # pylint: disable=protected-access + vhost = [x for x in self.config.parser.get_vhosts() + if 'geese.com' in x.names][0] + self.config._wildcard_vhosts["*.com"] = [vhost] + self.config.enhance("*.com", "staple-ocsp", "example/chain.pem") + self.assertFalse(mock_dialog.called) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_redirect_or_ocsp_no_install(self, mock_dialog): + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_dialog.return_value = [vhost] + self.config.enhance("*.com", "staple-ocsp", "example/chain.pem") + self.assertTrue(mock_dialog.called) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_double_redirect(self, mock_dialog): + # pylint: disable=protected-access + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + self.config._wildcard_redirect_vhosts["*.com"] = [vhost] + self.config.enhance("*.com", "redirect") + self.assertFalse(mock_dialog.called) + + def test_choose_vhosts_wildcard_no_ssl_filter_port(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [] + self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=False, + no_ssl_filter_port='80') + # Check that the dialog was called with only port 80 vhosts + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 4) + + class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" diff --git a/certbot-nginx/certbot_nginx/tests/display_ops_test.py b/certbot-nginx/certbot_nginx/tests/display_ops_test.py new file mode 100644 index 000000000..e3c6fb66b --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/display_ops_test.py @@ -0,0 +1,45 @@ +"""Test certbot_apache.display_ops.""" +import unittest + +from certbot.display import util as display_util + +from certbot.tests import util as certbot_util + +from certbot_nginx import parser + +from certbot_nginx.display_ops import select_vhost_multiple +from certbot_nginx.tests import util + + +class SelectVhostMultiTest(util.NginxTest): + """Tests for certbot_nginx.display_ops.select_vhost_multiple.""" + + def setUp(self): + super(SelectVhostMultiTest, self).setUp() + nparser = parser.NginxParser(self.config_path) + self.vhosts = nparser.get_vhosts() + + def test_select_no_input(self): + self.assertFalse(select_vhost_multiple([])) + + @certbot_util.patch_get_utility() + def test_select_correct(self, mock_util): + mock_util().checklist.return_value = ( + display_util.OK, [self.vhosts[3].display_repr(), + self.vhosts[2].display_repr()]) + vhs = select_vhost_multiple([self.vhosts[3], + self.vhosts[2], + self.vhosts[1]]) + self.assertTrue(self.vhosts[2] in vhs) + self.assertTrue(self.vhosts[3] in vhs) + self.assertFalse(self.vhosts[1] in vhs) + + @certbot_util.patch_get_utility() + def test_select_cancel(self, mock_util): + mock_util().checklist.return_value = (display_util.CANCEL, "whatever") + vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]]) + self.assertFalse(vhs) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 61ee293fa..72b65911c 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -61,10 +61,10 @@ class TlsSniPerformTest(util.NginxTest): shutil.rmtree(self.work_dir) @mock.patch("certbot_nginx.configurator" - ".NginxConfigurator.choose_vhost") + ".NginxConfigurator.choose_vhosts") def test_perform(self, mock_choose): self.sni.add_chall(self.achalls[1]) - mock_choose.return_value = None + mock_choose.return_value = [] result = self.sni.perform() self.assertFalse(result is None) diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index eca198bfe..0fd37e0cb 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -55,10 +55,11 @@ class NginxTlsSni01(common.TLSSNI01): self.configurator.config.tls_sni_01_port) for achall in self.achalls: - vhost = self.configurator.choose_vhost(achall.domain, create_if_no_match=True) + vhosts = self.configurator.choose_vhosts(achall.domain, create_if_no_match=True) - if vhost is not None and vhost.addrs: - addresses.append(list(vhost.addrs)) + # len is max 1 because Nginx doesn't authenticate wildcards + if vhosts and vhosts[0].addrs: + addresses.append(list(vhosts[0].addrs)) else: if ipv6: # If IPv6 is active in Nginx configuration From 8121acf2c1fed5514fd0a31a62f19dfbb92b2bb0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 14:54:48 -0800 Subject: [PATCH 78/93] Add user friendly wildcard error for ACMEv1 (#5636) * add WildcardUnsupportedError * Add friendly unsupported wildcard error msg * correct documentation * add version specifier --- acme/acme/client.py | 14 ++++++++++++++ acme/acme/client_test.py | 7 +++++++ acme/acme/errors.py | 3 +++ certbot/client.py | 30 +++++++++++++++++++++++------- certbot/tests/client_test.py | 2 +- 5 files changed, 48 insertions(+), 8 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index e3f6e845d..c6e897692 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -310,9 +310,17 @@ class Client(ClientBase): :returns: Authorization Resource. :rtype: `.AuthorizationResource` + :raises errors.WildcardUnsupportedError: if a wildcard is requested + """ if new_authzr_uri is not None: logger.debug("request_challenges with new_authzr_uri deprecated.") + + if identifier.value.startswith("*"): + raise errors.WildcardUnsupportedError( + "Requesting an authorization for a wildcard name is" + " forbidden by this version of the ACME protocol.") + new_authz = messages.NewAuthorization(identifier=identifier) response = self._post(self.directory.new_authz, new_authz) # TODO: handle errors @@ -333,6 +341,8 @@ class Client(ClientBase): :returns: Authorization Resource. :rtype: `.AuthorizationResource` + :raises errors.WildcardUnsupportedError: if a wildcard is requested + """ return self.request_challenges(messages.Identifier( typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) @@ -752,6 +762,10 @@ class BackwardsCompatibleClientV2(object): :returns: The newly created order. :rtype: OrderResource + + :raises errors.WildcardUnsupportedError: if a wildcard domain is + requested but unsupported by the ACME version + """ if self.acme_version == 1: csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 1e4db2884..060338360 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -376,6 +376,13 @@ class ClientTest(ClientTestBase): errors.UnexpectedUpdate, self.client.request_challenges, self.identifier) + def test_request_challenges_wildcard(self): + wildcard_identifier = messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value='*.example.org') + self.assertRaises( + errors.WildcardUnsupportedError, self.client.request_challenges, + wildcard_identifier) + def test_request_domain_challenges(self): self.client.request_challenges = mock.MagicMock() self.assertEqual( diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 991335958..97fa73614 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -115,3 +115,6 @@ class ConflictError(ClientError): self.location = location super(ConflictError, self).__init__() + +class WildcardUnsupportedError(Error): + """Error for when a wildcard is requested but is unsupported by ACME CA.""" diff --git a/certbot/client.py b/certbot/client.py index 81fc0b802..50d2262c4 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -12,6 +12,7 @@ import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util +from acme import errors as acme_errors from acme import messages import certbot @@ -258,10 +259,7 @@ class Client(object): logger.debug("CSR: %s", csr) if orderr is None: - orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr) - orderr = orderr.update(authorizations=authzr) - authzr = orderr.authorizations + orderr = self._get_order_and_authorizations(csr.data, best_effort=False) deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) orderr = self.acme.finalize_order(orderr, deadline) @@ -292,9 +290,8 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - orderr = self.acme.new_order(csr.data) - authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) - orderr = orderr.update(authorizations=authzr) + orderr = self._get_order_and_authorizations(csr.data, self.config.allow_subset_of_names) + authzr = orderr.authorizations auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] @@ -313,6 +310,25 @@ class Client(object): return cert, chain, key, csr + def _get_order_and_authorizations(self, csr_pem, best_effort): + """Request a new order and complete its authorizations. + + :param str csr_pem: A CSR in PEM format. + :param bool best_effort: True if failing to complete all + authorizations should not raise an exception + + :returns: order resource containing its completed authorizations + :rtype: acme.messages.OrderResource + + """ + try: + orderr = self.acme.new_order(csr_pem) + except acme_errors.WildcardUnsupportedError: + raise errors.Error("The currently selected ACME CA endpoint does" + " not support issuing wildcard certificates.") + authzr = self.auth_handler.handle_authorizations(orderr, best_effort) + return orderr.update(authorizations=authzr) + # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): """Obtain and enroll certificate. diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index b51275d9e..34595e463 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -184,7 +184,7 @@ class ClientTest(ClientTestCommon): self.client.obtain_certificate_from_csr( test_csr, orderr=None)) - auth_handler.handle_authorizations.assert_called_with(self.eg_order) + auth_handler.handle_authorizations.assert_called_with(self.eg_order, False) # Test for no auth_handler self.client.auth_handler = None From d8a54dc444a842e02c919b5092ebd745f25339e5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Mar 2018 14:55:45 -0800 Subject: [PATCH 79/93] Remove leading *. from default cert name. (#5639) --- certbot/client.py | 9 ++++++++- certbot/tests/client_test.py | 6 ++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 50d2262c4..eddf93e4f 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -354,7 +354,14 @@ class Client(object): "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") - new_name = certname if certname else domains[0] + if certname: + new_name = certname + elif util.is_wildcard_domain(domains[0]): + # Don't make files and directories starting with *. + new_name = domains[0][2:] + else: + new_name = domains[0] + if self.config.dry_run: logger.debug("Dry run: Skipping creating new lineage for %s", new_name) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 34595e463..5d01b103a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -285,7 +285,7 @@ class ClientTest(ClientTestCommon): @mock.patch('certbot.storage.RenewableCert.new_lineage') def test_obtain_and_enroll_certificate(self, mock_storage, mock_obtain_certificate): - domains = ["example.com", "www.example.com"] + domains = ["*.example.com", "example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), None) @@ -293,12 +293,14 @@ class ClientTest(ClientTestCommon): self.assertTrue(self.client.obtain_and_enroll_certificate(domains, "example_cert")) self.assertTrue(self.client.obtain_and_enroll_certificate(domains, None)) + self.assertTrue(self.client.obtain_and_enroll_certificate(domains[1:], None)) self.client.config.dry_run = True self.assertFalse(self.client.obtain_and_enroll_certificate(domains, None)) - self.assertTrue(mock_storage.call_count == 2) + names = [call[0][0] for call in mock_storage.call_args_list] + self.assertEqual(names, ["example_cert", "example.com", "example.com"]) @mock.patch("certbot.cli.helpful_parser") def test_save_certificate(self, mock_parser): From 8bc9cd67f0e6b445ee38342c904b8622c7f98878 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 1 Mar 2018 15:08:53 -0800 Subject: [PATCH 80/93] Fix ipv6only detection (#5648) * Fix ipv6only detection * move str() to inside ipv6_info * add regression test * Update to choose_vhosts --- certbot-nginx/certbot_nginx/configurator.py | 3 +++ .../certbot_nginx/tests/configurator_test.py | 12 ++++++++++++ .../testdata/etc_nginx/sites-enabled/ipv6ssl.com | 2 ++ 3 files changed, 17 insertions(+) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index e4d87744e..83e308bac 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -311,6 +311,9 @@ class NginxConfigurator(common.Installer): configuration, and existence of ipv6only directive for specified port :rtype: tuple of type (bool, bool) """ + # port should be a string, but it's easy to mess up, so let's + # make sure it is one + port = str(port) vhosts = self.parser.get_vhosts() ipv6_active = False ipv6only_present = False diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 722ba68bf..bffaef5e4 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -181,6 +181,18 @@ class NginxConfiguratorTest(util.NginxTest): # Port 443 has ipv6only=on because of ipv6ssl.com vhost self.assertEquals((True, True), self.config.ipv6_info("443")) + def test_ipv6only_detection(self): + self.config.version = (1, 3, 1) + + self.config.deploy_cert( + "ipv6.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + + for addr in self.config.choose_vhosts("ipv6.com")[0].addrs: + self.assertFalse(addr.ipv6only) def test_more_info(self): self.assertTrue('nginx.conf' in self.config.more_info()) diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com index d8f7eff12..875a9ee1b 100644 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com @@ -1,5 +1,7 @@ server { listen 443 ssl; listen [::]:443 ssl ipv6only=on; + listen 5001 ssl; + listen [::]:5001 ssl ipv6only=on; server_name ipv6ssl.com; } From e1878593d5608f908eebb262fe2c9c7dfcab55a1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Mar 2018 07:27:44 -0800 Subject: [PATCH 81/93] Ensure fullchain_pem in the order is unicode/str (#5654) * Decode fullchain_pem in ACMEv1 * Convert back to bytes in Certbot * document bytes are returned --- acme/acme/client.py | 4 ++-- acme/acme/client_test.py | 4 ++-- acme/acme/crypto_util.py | 3 +++ certbot/client.py | 5 +++-- certbot/crypto_util.py | 3 ++- certbot/tests/client_test.py | 22 ++++++++++++---------- certbot/tests/crypto_util_test.py | 4 ++-- 7 files changed, 26 insertions(+), 19 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c6e897692..d52c82a5c 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -809,8 +809,8 @@ class BackwardsCompatibleClientV2(object): 'certificate, please rerun the command for a new one.') cert = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) - chain = crypto_util.dump_pyopenssl_chain(chain) + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped).decode() + chain = crypto_util.dump_pyopenssl_chain(chain).decode() return orderr.update(fullchain_pem=(cert + chain)) else: diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 060338360..a0c27e74f 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -99,10 +99,10 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): self.chain = [wrapped, wrapped] self.cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped) + OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped).decode() single_chain = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, loaded) + OpenSSL.crypto.FILETYPE_PEM, loaded).decode() self.chain_pem = single_chain + single_chain self.fullchain_pem = self.cert_pem + self.chain_pem diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 07b55ae33..2281196eb 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -287,6 +287,9 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in :class:`josepy.util.ComparableX509`). + :returns: certificate chain bundle + :rtype: bytes + """ # XXX: returns empty string when no chain is available, which # shuts up RenewableCert, but might not be the best solution... diff --git a/certbot/client.py b/certbot/client.py index eddf93e4f..2992c0cec 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -244,7 +244,7 @@ class Client(object): than `authkey`. :param acme.messages.OrderResource orderr: contains authzrs - :returns: certificate and chain as PEM strings + :returns: certificate and chain as PEM byte strings :rtype: tuple """ @@ -263,7 +263,8 @@ class Client(object): deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) orderr = self.acme.finalize_order(orderr, deadline) - return crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) + cert, chain = crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) + return cert.encode(), chain.encode() def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 11721cc10..37118c591 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -441,8 +441,9 @@ def cert_and_chain_from_fullchain(fullchain_pem): :returns: tuple of string cert_pem and chain_pem :rtype: tuple + """ cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)).decode() chain = fullchain_pem[len(cert):] return (cert, chain) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 5d01b103a..0f2c58161 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -132,7 +132,6 @@ class ClientTest(ClientTestCommon): self.eg_domains = ["example.com", "www.example.com"] self.eg_order = mock.MagicMock( authorizations=[None], - fullchain_pem=mock.sentinel.fullchain_pem, csr_pem=mock.sentinel.csr_pem) def test_init_acme_verify_ssl(self): @@ -165,8 +164,7 @@ class ClientTest(ClientTestCommon): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler - mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) orderr = self.acme.new_order(test_csr.data) auth_handler.handle_authorizations(orderr, False) @@ -199,8 +197,7 @@ class ClientTest(ClientTestCommon): csr = util.CSR(form="pem", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key - mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) self._test_obtain_certificate_common(mock.sentinel.key, csr) @@ -209,7 +206,7 @@ class ClientTest(ClientTestCommon): mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with( - mock.sentinel.fullchain_pem) + self.eg_order.fullchain_pem) @mock.patch("certbot.client.crypto_util") @mock.patch("os.remove") @@ -218,8 +215,7 @@ class ClientTest(ClientTestCommon): key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = key - mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) authzr = self._authzr_from_domains(["example.com"]) self.config.allow_subset_of_names = True @@ -237,8 +233,7 @@ class ClientTest(ClientTestCommon): mock_acme_crypto.make_csr.return_value = CSR_SAN mock_crypto.make_key.return_value = mock.sentinel.key_pem key = util.Key(file=None, pem=mock.sentinel.key_pem) - mock_crypto.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, - mock.sentinel.chain) + self._set_mock_from_fullchain(mock_crypto.cert_and_chain_from_fullchain) self.client.config.dry_run = True self._test_obtain_certificate_common(key, csr) @@ -250,6 +245,13 @@ class ClientTest(ClientTestCommon): mock_crypto.init_save_csr.assert_not_called() self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1) + def _set_mock_from_fullchain(self, mock_from_fullchain): + mock_cert = mock.Mock() + mock_cert.encode.return_value = mock.sentinel.cert + mock_chain = mock.Mock() + mock_chain.encode.return_value = mock.sentinel.chain + mock_from_fullchain.return_value = (mock_cert, mock_chain) + def _authzr_from_domains(self, domains): authzr = [] diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 00303fab3..480139378 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -377,8 +377,8 @@ class CertAndChainFromFullchainTest(unittest.TestCase): """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" def test_cert_and_chain_from_fullchain(self): - cert_pem = CERT - chain_pem = CERT + SS_CERT + cert_pem = CERT.decode() + chain_pem = cert_pem + SS_CERT.decode() fullchain_pem = cert_pem + chain_pem from certbot.crypto_util import cert_and_chain_from_fullchain cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem) From cc344bfd1e080ad8ae253e0b1073a5ba7879583d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Mar 2018 09:50:19 -0800 Subject: [PATCH 82/93] Break lockstep between our packages (#5655) Fixes #5490. There's a lot of possibilities discussed in #5490, but I'll try and explain what I actually did here as succinctly as I can. Unfortunately, there's a fair bit to explain. My goal was to break lockstep and give us tests to ensure the minimum specified versions are correct without taking the time now to refactor our whole test setup. To handle specifying each package's minimum acme/certbot version, I added a requirements file to each package. This won't actually be included in the shipped package (because it's not in the MANIFEST). After creating these files and modifying tools/pip_install.sh to use them, I created a separate tox env for most packages (I kept the DNS plugins together for convenience). The reason this is necessary is because we currently use a single environment for each plugin, but if we used this approach for these tests we'd hit issues due to different installed plugins requiring different versions of acme/certbot. There's a lot more discussion about this in #5490 if you're interested in this piece. I unfortunately wasted a lot of time trying to remove the boilerplate this approach causes in tox.ini, but to do this I think we need negations described at complex factor conditions which hasn't made it into a tox release yet. The biggest missing piece here is how to make sure the oldest versions that are currently pinned to master get updated. Currently, they'll stay pinned that way without manual intervention and won't be properly testing the oldest version. I think we should solve this during the larger test/repo refactoring after the release because the tests are using the correct values now and I don't see a simple way around the problem. Once this lands, I'm planning on updating the test-everything tests to do integration tests with the "oldest" versions here. * break lockstep between packages * Use per package requirements files * add local oldest requirements files * update tox.ini * work with dev0 versions * Install requirements in separate step. * don't error when we don't have requirements * install latest packages in editable mode * Update .travis.yml * Add reminder comments * move dev to requirements * request acme[dev] * Update pip_install documentation --- .travis.yml | 2 +- certbot-apache/local-oldest-requirements.txt | 2 + certbot-apache/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-cloudflare/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-cloudxns/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-digitalocean/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-dnsimple/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-dnsmadeeasy/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-google/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-luadns/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-nsone/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-rfc2136/setup.py | 7 +-- .../local-oldest-requirements.txt | 2 + certbot-dns-route53/setup.py | 6 ++- certbot-nginx/local-oldest-requirements.txt | 2 + certbot-nginx/setup.py | 10 ++-- local-oldest-requirements.txt | 1 + setup.py | 4 +- tools/pip_install.sh | 34 +++++++++--- tox.ini | 53 +++++++++++++++++-- 29 files changed, 154 insertions(+), 50 deletions(-) create mode 100644 certbot-apache/local-oldest-requirements.txt create mode 100644 certbot-dns-cloudflare/local-oldest-requirements.txt create mode 100644 certbot-dns-cloudxns/local-oldest-requirements.txt create mode 100644 certbot-dns-digitalocean/local-oldest-requirements.txt create mode 100644 certbot-dns-dnsimple/local-oldest-requirements.txt create mode 100644 certbot-dns-dnsmadeeasy/local-oldest-requirements.txt create mode 100644 certbot-dns-google/local-oldest-requirements.txt create mode 100644 certbot-dns-luadns/local-oldest-requirements.txt create mode 100644 certbot-dns-nsone/local-oldest-requirements.txt create mode 100644 certbot-dns-rfc2136/local-oldest-requirements.txt create mode 100644 certbot-dns-route53/local-oldest-requirements.txt create mode 100644 certbot-nginx/local-oldest-requirements.txt create mode 100644 local-oldest-requirements.txt diff --git a/.travis.yml b/.travis.yml index c62664180..9ec2f724b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ matrix: - python: "2.7" env: TOXENV=lint - python: "2.7" - env: TOXENV=py27-oldest + env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest' sudo: required services: docker - python: "3.4" diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-apache/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 336233bd4..7608c0647 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'mock', 'python-augeas', 'setuptools', diff --git a/certbot-dns-cloudflare/local-oldest-requirements.txt b/certbot-dns-cloudflare/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-cloudflare/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index e5687a9f5..4ed8e796d 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'cloudflare>=1.5.1', 'mock', 'setuptools', diff --git a/certbot-dns-cloudxns/local-oldest-requirements.txt b/certbot-dns-cloudxns/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-cloudxns/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 0ef31a90c..7f973709c 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-digitalocean/local-oldest-requirements.txt b/certbot-dns-digitalocean/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-digitalocean/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 11c2aea24..0ce91e64e 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'mock', 'python-digitalocean>=1.11', 'setuptools', diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 414a058fa..d12b26d83 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 18d773347..856eaba0f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-google/local-oldest-requirements.txt b/certbot-dns-google/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-google/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index d5def1bf9..0dfff0402 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', # 1.5 is the first version that supports oauth2client>=2.0 'google-api-python-client>=1.5', 'mock', diff --git a/certbot-dns-luadns/local-oldest-requirements.txt b/certbot-dns-luadns/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-luadns/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 13fa742d5..b255691dc 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-nsone/local-oldest-requirements.txt b/certbot-dns-nsone/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-nsone/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 01c9579c1..68d8f6cdb 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', 'setuptools', diff --git a/certbot-dns-rfc2136/local-oldest-requirements.txt b/certbot-dns-rfc2136/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-rfc2136/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 64b126595..3d6b3799b 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -6,10 +6,11 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dnspython', 'mock', 'setuptools', diff --git a/certbot-dns-route53/local-oldest-requirements.txt b/certbot-dns-route53/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-route53/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index e45343f79..ad20725b5 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -5,9 +5,11 @@ from setuptools import find_packages version = '0.22.0.dev0' +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'boto3', 'mock', 'setuptools', diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt new file mode 100644 index 000000000..65f5a758e --- /dev/null +++ b/certbot-nginx/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +-e acme[dev] +-e .[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 96f8b834d..bb71cf19a 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -6,10 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + # This plugin works with an older version of acme, but Certbot does not. + # 0.22.0 is specified here to work around + # https://github.com/pypa/pip/issues/988. + 'acme>0.21.1', + 'certbot>0.21.1', 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt new file mode 100644 index 000000000..2346300a3 --- /dev/null +++ b/local-oldest-requirements.txt @@ -0,0 +1 @@ +-e acme[dev] diff --git a/setup.py b/setup.py index 9ac1a7ee7..3667a6976 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,9 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme=={0}'.format(version), + # Remember to update local-oldest-requirements.txt when changing the + # minimum acme version. + 'acme>0.21.1', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. diff --git a/tools/pip_install.sh b/tools/pip_install.sh index d2aae4a43..b385c5482 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,18 +1,30 @@ #!/bin/bash -e # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt and -# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's -# requirements file and tools/dev_constraints.txt is used. The other file -# always takes precedence over tools/dev_constraints.txt. +# to 1, a combination of tools/oldest_constraints.txt, +# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the +# top level of the package's directory is used, otherwise, a combination of +# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# other file always takes precedence over tools/dev_constraints.txt. If +# CERTBOT_OLDEST is set, this script must be run with `-e ` and +# no other arguments. # get the root of the Certbot repo tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -dev_constraints="$tools_dir/dev_constraints.txt" -merge_reqs="$tools_dir/merge_requirements.py" +all_constraints=$(mktemp) test_constraints=$(mktemp) -trap "rm -f $test_constraints" EXIT +trap "rm -f $all_constraints $test_constraints" EXIT if [ "$CERTBOT_OLDEST" = 1 ]; then + if [ "$1" != "-e" -o "$#" -ne "2" ]; then + echo "When CERTBOT_OLDEST is set, this script must be run with a single -e argument." + exit 1 + fi + pkg_dir=$(echo $2 | cut -f1 -d\[) # remove any extras such as [dev] + requirements="$pkg_dir/local-oldest-requirements.txt" + # packages like acme don't have any local oldest requirements + if [ ! -f "$requirements" ]; then + unset requirements + fi cp "$tools_dir/oldest_constraints.txt" "$test_constraints" else repo_root=$(dirname "$tools_dir") @@ -20,7 +32,13 @@ else sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" fi +"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \ + "$test_constraints" > "$all_constraints" + set -x # install the requested packages using the pinned requirements as constraints -pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@" +if [ -n "$requirements" ]; then + pip install -q --constraint "$all_constraints" --requirement "$requirements" +fi +pip install -q --constraint "$all_constraints" "$@" diff --git a/tox.ini b/tox.ini index 971aa7631..049220bbb 100644 --- a/tox.ini +++ b/tox.ini @@ -14,10 +14,7 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -all_packages = - acme[dev] \ - .[dev] \ - certbot-apache \ +dns_packages = certbot-dns-cloudflare \ certbot-dns-cloudxns \ certbot-dns-digitalocean \ @@ -27,7 +24,12 @@ all_packages = certbot-dns-luadns \ certbot-dns-nsone \ certbot-dns-rfc2136 \ - certbot-dns-route53 \ + certbot-dns-route53 +all_packages = + acme[dev] \ + .[dev] \ + certbot-apache \ + {[base]dns_packages} \ certbot-nginx \ letshelp-certbot install_packages = @@ -70,6 +72,47 @@ setenv = passenv = {[testenv]passenv} +[testenv:py27-acme-oldest] +commands = + {[base]install_and_test} acme[dev] +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-apache-oldest] +commands = + {[base]install_and_test} certbot-apache +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-certbot-oldest] +commands = + {[base]install_and_test} .[dev] +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-dns-oldest] +commands = + {[base]install_and_test} {[base]dns_packages} +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-nginx-oldest] +commands = + {[base]install_and_test} certbot-nginx + python tests/lock_test.py +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + [testenv:py27_install] basepython = python2.7 commands = From 441625c6102126f2d63daa964ecac4073e583d0a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 5 Mar 2018 22:49:02 +0200 Subject: [PATCH 83/93] Allow Google DNS plugin to write multiple TXT record values (#5652) * Allow Google DNS plugin to write multiple TXT record values in same resourcerecord * Atomic updates * Split rrsets request --- .../certbot_dns_google/dns_google.py | 63 ++++++++++++++++++- .../certbot_dns_google/dns_google_test.py | 61 +++++++++++++++++- 2 files changed, 119 insertions(+), 5 deletions(-) diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index cea754c06..ab8bf20de 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -107,6 +107,15 @@ class _GoogleClient(object): zone_id = self._find_managed_zone_id(domain) + record_contents = self.get_existing_txt_rrset(zone_id, record_name) + add_records = record_contents[:] + + if "\""+record_content+"\"" in record_contents: + # The process was interrupted previously and validation token exists + return + + add_records.append(record_content) + data = { "kind": "dns#change", "additions": [ @@ -114,12 +123,24 @@ class _GoogleClient(object): "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": [record_content, ], + "rrdatas": add_records, "ttl": record_ttl, }, ], } + if record_contents: + # We need to remove old records in the same request + data["deletions"] = [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": record_name + ".", + "rrdatas": record_contents, + "ttl": record_ttl, + }, + ] + changes = self.dns.changes() # changes | pylint: disable=no-member try: @@ -154,6 +175,8 @@ class _GoogleClient(object): logger.warn('Error finding zone. Skipping cleanup.') return + record_contents = self.get_existing_txt_rrset(zone_id, record_name) + data = { "kind": "dns#change", "deletions": [ @@ -161,12 +184,26 @@ class _GoogleClient(object): "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": [record_content, ], + "rrdatas": record_contents, "ttl": record_ttl, }, ], } + # Remove the record being deleted from the list + readd_contents = [r for r in record_contents if r != "\"" + record_content + "\""] + if readd_contents: + # We need to remove old records in the same request + data["additions"] = [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": record_name + ".", + "rrdatas": readd_contents, + "ttl": record_ttl, + }, + ] + changes = self.dns.changes() # changes | pylint: disable=no-member try: @@ -175,6 +212,28 @@ class _GoogleClient(object): except googleapiclient_errors.Error as e: logger.warn('Encountered error deleting TXT record: %s', e) + def get_existing_txt_rrset(self, zone_id, record_name): + """ + Get existing TXT records from the RRset for the record name. + + :param str zone_id: The ID of the managed zone. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + + :returns: List of TXT record values + :rtype: `list` of `string` + + """ + rrs_request = self.dns.resourceRecordSets() # pylint: disable=no-member + request = rrs_request.list(managedZone=zone_id, project=self.project_id) + response = request.execute() + # Add dot as the API returns absolute domains + record_name += "." + if response: + for rr in response["rrsets"]: + if rr["name"] == record_name and rr["type"] == "TXT": + return rr["rrdatas"] + return [] + def _find_managed_zone_id(self, domain): """ Find the managed zone for a given domain. diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 53f84dd6e..3291b2c3a 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -74,10 +74,15 @@ class GoogleClientTest(unittest.TestCase): mock_mz = mock.MagicMock() mock_mz.list.return_value.execute.side_effect = zone_request_side_effect + mock_rrs = mock.MagicMock() + rrsets = {"rrsets": [{"name": "_acme-challenge.example.org.", "type": "TXT", + "rrdatas": ["\"example-txt-contents\""]}]} + mock_rrs.list.return_value.execute.return_value = rrsets mock_changes = mock.MagicMock() client.dns.managedZones = mock.MagicMock(return_value=mock_mz) client.dns.changes = mock.MagicMock(return_value=mock_changes) + client.dns.resourceRecordSets = mock.MagicMock(return_value=mock_rrs) return client, mock_changes @@ -137,6 +142,30 @@ class GoogleClientTest(unittest.TestCase): managedZone=self.zone, project=PROJECT_ID) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_delete_old(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + mock_rrs.return_value = ["sample-txt-contents"] + client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + self.assertTrue(changes.create.called) + self.assertTrue("sample-txt-contents" in + changes.create.call_args_list[0][1]["body"]["deletions"][0]["rrdatas"]) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_noop(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + client.add_txt_record(DOMAIN, "_acme-challenge.example.org", + "example-txt-contents", self.record_ttl) + self.assertFalse(changes.create.called) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) @@ -172,7 +201,12 @@ class GoogleClientTest(unittest.TestCase): def test_del_txt_record(self, unused_credential_mock): client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) - client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + mock_rrs.return_value = ["\"sample-txt-contents\"", + "\"example-txt-contents\""] + client.del_txt_record(DOMAIN, "_acme-challenge.example.org", + "example-txt-contents", self.record_ttl) expected_body = { "kind": "dns#change", @@ -180,8 +214,17 @@ class GoogleClientTest(unittest.TestCase): { "kind": "dns#resourceRecordSet", "type": "TXT", - "name": self.record_name + ".", - "rrdatas": [self.record_content, ], + "name": "_acme-challenge.example.org.", + "rrdatas": ["\"sample-txt-contents\"", "\"example-txt-contents\""], + "ttl": self.record_ttl, + }, + ], + "additions": [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": "_acme-challenge.example.org.", + "rrdatas": ["\"sample-txt-contents\"", ], "ttl": self.record_ttl, }, ], @@ -217,6 +260,18 @@ class GoogleClientTest(unittest.TestCase): client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + # Record name mocked in setUp + found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") + self.assertEquals(found, ["\"example-txt-contents\""]) + not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") + self.assertEquals(not_found, []) + def test_get_project_id(self): from certbot_dns_google.dns_google import _GoogleClient From fe682e779b82ab0dfd72342369df630495c26a20 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Sun, 4 Mar 2018 16:24:42 +0200 Subject: [PATCH 84/93] ACMEv2 support for Route53 plugin --- .../certbot_dns_route53/dns_route53.py | 26 +++++++-- .../certbot_dns_route53/dns_route53_test.py | 54 +++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index 67462e369..c0e8e5495 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -85,9 +85,29 @@ class Authenticator(dns_common.DNSAuthenticator): zones.sort(key=lambda z: len(z[0]), reverse=True) return zones[0][1] + def _get_validation_rrset(self, zone_id, validation_domain_name): + validation_domain_name += "." + records = self.r53.list_resource_record_sets(HostedZoneId=zone_id) + for record in records["ResourceRecordSets"]: + if record["Name"] == validation_domain_name and record["Type"] == "TXT": + return record["ResourceRecords"] + return [] + def _change_txt_record(self, action, validation_domain_name, validation): zone_id = self._find_zone_id_for_domain(validation_domain_name) + rrecords = self._get_validation_rrset(zone_id, validation_domain_name) + challenge = {"Value": '"{0}"'.format(validation)} + if action == "DELETE": + if len(rrecords) > 1: + # Need to update instead, as we're not deleting the rrset + action = "UPSERT" + # Remove the record being deleted from the list + rrecords = [rr for rr in rrecords if rr != challenge] + else: + if challenge not in rrecords: + rrecords.append(challenge) + response = self.r53.change_resource_record_sets( HostedZoneId=zone_id, ChangeBatch={ @@ -99,11 +119,7 @@ class Authenticator(dns_common.DNSAuthenticator): "Name": validation_domain_name, "Type": "TXT", "TTL": self.ttl, - "ResourceRecords": [ - # For some reason TXT records need to be - # manually quoted. - {"Value": '"{0}"'.format(validation)} - ], + "ResourceRecords": rrecords, } } ] diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index d5f1b2816..9aec05b6e 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -178,6 +178,9 @@ class ClientTest(unittest.TestCase): def test_change_txt_record(self): self.client._find_zone_id_for_domain = mock.MagicMock() + self.client._get_validation_rrset = mock.MagicMock( + return_value=[] + ) self.client.r53.change_resource_record_sets = mock.MagicMock( return_value={"ChangeInfo": {"Id": 1}}) @@ -186,6 +189,57 @@ class ClientTest(unittest.TestCase): call_count = self.client.r53.change_resource_record_sets.call_count self.assertEqual(call_count, 1) + def test_change_txt_record_multirecord(self): + self.client._find_zone_id_for_domain = mock.MagicMock() + self.client._get_validation_rrset = mock.MagicMock() + self.client._get_validation_rrset.return_value = [ + {"Value": "\"pre-existing-value\""}, + {"Value": "\"pre-existing-value-two\""}, + ] + self.client.r53.change_resource_record_sets = mock.MagicMock( + return_value={"ChangeInfo": {"Id": 1}}) + + self.client._change_txt_record("DELETE", DOMAIN, "pre-existing-value") + + call_count = self.client.r53.change_resource_record_sets.call_count + call_args = self.client.r53.change_resource_record_sets.call_args_list[0][1] + call_args_batch = call_args["ChangeBatch"]["Changes"][0] + self.assertEqual(call_args_batch["Action"], "UPSERT") + self.assertEqual( + call_args_batch["ResourceRecordSet"]["ResourceRecords"], + [{"Value": "\"pre-existing-value-two\""}]) + + self.assertEqual(call_count, 1) + + def test_get_validation_rrset(self): + self.client.r53.list_resource_record_sets = mock.MagicMock( + return_value={"ResourceRecordSets": [ + {"Name": "_acme-challenge.example.org.", + "Type": "TXT", + "ResourceRecords": [ + {"Value": "\"validation-token\""}, + {"Value": "\"another-validation-token\""}, + ], + }, + {"Name": "_acme-challenge.example.org.", + "Type": "NS", + "ResourceRecords": [ + {"Value": "ns1.example.com"}, + ], + } + ]}) + rrset = self.client._get_validation_rrset("zoneid", + "_acme-challenge.example.org") + self.assertEquals(len(rrset), 2) + self.assertTrue({"Value": "\"another-validation-token\""} in rrset) + + def test_get_validation_rrset_empty(self): + self.client.r53.list_resource_record_sets = mock.MagicMock( + return_value={"ResourceRecordSets": []}) + rrset = self.client._get_validation_rrset("zoneid", + "_acme-challenge.example.org") + self.assertEquals(rrset, []) + def test_wait_for_change(self): self.client.r53.get_change = mock.MagicMock( side_effect=[{"ChangeInfo": {"Status": "PENDING"}}, From 7bc45121a13537cceef4e4bf53d4738925d55511 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Mar 2018 18:36:03 -0800 Subject: [PATCH 85/93] Remove the need for route53:ListResourceRecordSets * add test_change_txt_record_delete --- .../certbot_dns_route53/dns_route53.py | 24 ++++----- .../certbot_dns_route53/dns_route53_test.py | 54 ++++++++----------- 2 files changed, 31 insertions(+), 47 deletions(-) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index c0e8e5495..08b1d03f0 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -1,4 +1,5 @@ """Certbot Route53 authenticator plugin.""" +import collections import logging import time @@ -33,6 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) self.r53 = boto3.client("route53") + self._resource_records = collections.defaultdict(list) def more_info(self): # pylint: disable=missing-docstring,no-self-use return "Solve a DNS01 challenge using AWS Route53" @@ -85,28 +87,22 @@ class Authenticator(dns_common.DNSAuthenticator): zones.sort(key=lambda z: len(z[0]), reverse=True) return zones[0][1] - def _get_validation_rrset(self, zone_id, validation_domain_name): - validation_domain_name += "." - records = self.r53.list_resource_record_sets(HostedZoneId=zone_id) - for record in records["ResourceRecordSets"]: - if record["Name"] == validation_domain_name and record["Type"] == "TXT": - return record["ResourceRecords"] - return [] - def _change_txt_record(self, action, validation_domain_name, validation): zone_id = self._find_zone_id_for_domain(validation_domain_name) - rrecords = self._get_validation_rrset(zone_id, validation_domain_name) + rrecords = self._resource_records[validation_domain_name] challenge = {"Value": '"{0}"'.format(validation)} if action == "DELETE": - if len(rrecords) > 1: + # Remove the record being deleted from the list of tracked records + rrecords.remove(challenge) + if rrecords: # Need to update instead, as we're not deleting the rrset action = "UPSERT" - # Remove the record being deleted from the list - rrecords = [rr for rr in rrecords if rr != challenge] + else: + # Create a new list containing the record to use with DELETE + rrecords = [challenge] else: - if challenge not in rrecords: - rrecords.append(challenge) + rrecords.append(challenge) response = self.r53.change_resource_record_sets( HostedZoneId=zone_id, diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index 9aec05b6e..7534e132c 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -178,9 +178,6 @@ class ClientTest(unittest.TestCase): def test_change_txt_record(self): self.client._find_zone_id_for_domain = mock.MagicMock() - self.client._get_validation_rrset = mock.MagicMock( - return_value=[] - ) self.client.r53.change_resource_record_sets = mock.MagicMock( return_value={"ChangeInfo": {"Id": 1}}) @@ -189,10 +186,30 @@ class ClientTest(unittest.TestCase): call_count = self.client.r53.change_resource_record_sets.call_count self.assertEqual(call_count, 1) + def test_change_txt_record_delete(self): + self.client._find_zone_id_for_domain = mock.MagicMock() + self.client.r53.change_resource_record_sets = mock.MagicMock( + return_value={"ChangeInfo": {"Id": 1}}) + + validation = "some-value" + validation_record = {"Value": '"{0}"'.format(validation)} + self.client._resource_records[DOMAIN] = [validation_record] + + self.client._change_txt_record("DELETE", DOMAIN, validation) + + call_count = self.client.r53.change_resource_record_sets.call_count + self.assertEqual(call_count, 1) + call_args = self.client.r53.change_resource_record_sets.call_args_list[0][1] + call_args_batch = call_args["ChangeBatch"]["Changes"][0] + self.assertEqual(call_args_batch["Action"], "DELETE") + self.assertEqual( + call_args_batch["ResourceRecordSet"]["ResourceRecords"], + [validation_record]) + def test_change_txt_record_multirecord(self): self.client._find_zone_id_for_domain = mock.MagicMock() self.client._get_validation_rrset = mock.MagicMock() - self.client._get_validation_rrset.return_value = [ + self.client._resource_records[DOMAIN] = [ {"Value": "\"pre-existing-value\""}, {"Value": "\"pre-existing-value-two\""}, ] @@ -211,35 +228,6 @@ class ClientTest(unittest.TestCase): self.assertEqual(call_count, 1) - def test_get_validation_rrset(self): - self.client.r53.list_resource_record_sets = mock.MagicMock( - return_value={"ResourceRecordSets": [ - {"Name": "_acme-challenge.example.org.", - "Type": "TXT", - "ResourceRecords": [ - {"Value": "\"validation-token\""}, - {"Value": "\"another-validation-token\""}, - ], - }, - {"Name": "_acme-challenge.example.org.", - "Type": "NS", - "ResourceRecords": [ - {"Value": "ns1.example.com"}, - ], - } - ]}) - rrset = self.client._get_validation_rrset("zoneid", - "_acme-challenge.example.org") - self.assertEquals(len(rrset), 2) - self.assertTrue({"Value": "\"another-validation-token\""} in rrset) - - def test_get_validation_rrset_empty(self): - self.client.r53.list_resource_record_sets = mock.MagicMock( - return_value={"ResourceRecordSets": []}) - rrset = self.client._get_validation_rrset("zoneid", - "_acme-challenge.example.org") - self.assertEquals(rrset, []) - def test_wait_for_change(self): self.client.r53.get_change = mock.MagicMock( side_effect=[{"ChangeInfo": {"Status": "PENDING"}}, From cee9ac586ea43d355ede8eac71f5d145902169a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Mar 2018 07:20:34 -0800 Subject: [PATCH 86/93] Don't report coverage on Apache during integration tests (#5669) * ignore Apache coverage * drop min coverage to 67 --- tests/boulder-integration.sh | 2 +- tests/integration/_common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index f2b0dcf60..b5a305016 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -443,4 +443,4 @@ then . ./certbot-nginx/tests/boulder-integration.sh fi -coverage report --fail-under 63 -m +coverage report --fail-under 67 -m diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 236090a14..a8d35ed89 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -23,7 +23,7 @@ fi certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" - omit_patterns="$omit_patterns,*_test.py,*_test_*," + omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*" omit_patterns="$omit_patterns,certbot-compatibility-test/*,certbot-dns*/" coverage run \ --append \ From d62c56f9c91a920a07f338bbc7aa53b7329624ac Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Mar 2018 07:21:01 -0800 Subject: [PATCH 87/93] Remove the assumption the domain is unique in the manual plugin (#5670) * use entire achall as key * Add manual cleanup hook * use manual cleanup hook --- certbot/plugins/manual.py | 4 ++-- certbot/plugins/manual_test.py | 6 +++--- tests/boulder-integration.sh | 4 +++- tests/manual-dns-cleanup.sh | 3 +++ 4 files changed, 11 insertions(+), 6 deletions(-) create mode 100755 tests/manual-dns-cleanup.sh diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 07371ad34..614449d34 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -189,7 +189,7 @@ when it receives a TLS ClientHello with the SNI extension set to os.environ.update(env) _, out = hooks.execute(self.conf('auth-hook')) env['CERTBOT_AUTH_OUTPUT'] = out.strip() - self.env[achall.domain] = env + self.env[achall] = env def _perform_achall_manually(self, achall): validation = achall.validation(achall.account_key) @@ -215,7 +215,7 @@ when it receives a TLS ClientHello with the SNI extension set to def cleanup(self, achalls): # pylint: disable=missing-docstring if self.conf('cleanup-hook'): for achall in achalls: - env = self.env.pop(achall.domain) + env = self.env.pop(achall) if 'CERTBOT_TOKEN' not in env: os.environ.pop('CERTBOT_TOKEN', None) os.environ.update(env) diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index ac528e81c..e5c22b377 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -93,10 +93,10 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.auth.perform(self.achalls), [achall.response(achall.account_key) for achall in self.achalls]) self.assertEqual( - self.auth.env[self.dns_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.dns_achall]['CERTBOT_AUTH_OUTPUT'], dns_expected) self.assertEqual( - self.auth.env[self.http_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.http_achall]['CERTBOT_AUTH_OUTPUT'], http_expected) # tls_sni_01 challenge must be perform()ed above before we can # get the cert_path and key_path. @@ -107,7 +107,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.auth.tls_sni_01.get_z_domain(self.tls_sni_achall), 'novalidation') self.assertEqual( - self.auth.env[self.tls_sni_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.tls_sni_achall]['CERTBOT_AUTH_OUTPUT'], tls_sni_expected) @test_util.patch_get_utility() diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index b5a305016..2b92476fd 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -233,6 +233,7 @@ certname="dns.le.wtf" common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ --cert-name $certname \ --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ --renew-hook 'echo deploy >> "$HOOK_TEST"' @@ -433,7 +434,8 @@ done # Test ACMEv2-only features if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ - --manual-auth-hook ./tests/manual-dns-auth.sh + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh fi # Most CI systems set this variable to true. diff --git a/tests/manual-dns-cleanup.sh b/tests/manual-dns-cleanup.sh new file mode 100755 index 000000000..0c5c56b17 --- /dev/null +++ b/tests/manual-dns-cleanup.sh @@ -0,0 +1,3 @@ +#!/bin/sh +curl -X POST 'http://localhost:8055/clear-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}" From 6357e051f4841df8af69a8abf5a4ba3dc8578c3c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Mar 2018 15:32:22 -0800 Subject: [PATCH 88/93] Fallback without dns.resourceRecordSets.list permission (#5678) * Add rrset list fallback * List dns.resourceRecordSets.list as required * Handle list failures differently for add and del * Quote record content * disable not-callable for iter_entry_points * List update permission --- .../certbot_dns_google/__init__.py | 2 ++ .../certbot_dns_google/dns_google.py | 29 ++++++++++++++----- .../certbot_dns_google/dns_google_test.py | 14 ++++++++- certbot/plugins/disco.py | 1 + 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py index 7349a7696..f19266737 100644 --- a/certbot-dns-google/certbot_dns_google/__init__.py +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -29,6 +29,8 @@ for an account with the following permissions: * ``dns.managedZones.list`` * ``dns.resourceRecordSets.create`` * ``dns.resourceRecordSets.delete`` +* ``dns.resourceRecordSets.list`` +* ``dns.resourceRecordSets.update`` Google provides instructions for `creating a service account `_ and diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index ab8bf20de..e2088b357 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -108,6 +108,8 @@ class _GoogleClient(object): zone_id = self._find_managed_zone_id(domain) record_contents = self.get_existing_txt_rrset(zone_id, record_name) + if record_contents is None: + record_contents = [] add_records = record_contents[:] if "\""+record_content+"\"" in record_contents: @@ -176,6 +178,8 @@ class _GoogleClient(object): return record_contents = self.get_existing_txt_rrset(zone_id, record_name) + if record_contents is None: + record_contents = ["\"" + record_content + "\""] data = { "kind": "dns#change", @@ -216,23 +220,32 @@ class _GoogleClient(object): """ Get existing TXT records from the RRset for the record name. + If an error occurs while requesting the record set, it is suppressed + and None is returned. + :param str zone_id: The ID of the managed zone. :param str record_name: The record name (typically beginning with '_acme-challenge.'). - :returns: List of TXT record values - :rtype: `list` of `string` + :returns: List of TXT record values or None + :rtype: `list` of `string` or `None` """ rrs_request = self.dns.resourceRecordSets() # pylint: disable=no-member request = rrs_request.list(managedZone=zone_id, project=self.project_id) - response = request.execute() # Add dot as the API returns absolute domains record_name += "." - if response: - for rr in response["rrsets"]: - if rr["name"] == record_name and rr["type"] == "TXT": - return rr["rrdatas"] - return [] + try: + response = request.execute() + except googleapiclient_errors.Error: + logger.info("Unable to list existing records. If you're " + "requesting a wildcard certificate, this might not work.") + logger.debug("Error was:", exc_info=True) + else: + if response: + for rr in response["rrsets"]: + if rr["name"] == record_name and rr["type"] == "TXT": + return rr["rrdatas"] + return None def _find_managed_zone_id(self, domain): """ diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 3291b2c3a..afab847cf 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -270,7 +270,19 @@ class GoogleClientTest(unittest.TestCase): found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") self.assertEquals(found, ["\"example-txt-contents\""]) not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") - self.assertEquals(not_found, []) + self.assertEquals(not_found, None) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing_fallback(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + mock_execute = client.dns.resourceRecordSets.return_value.list.return_value.execute + mock_execute.side_effect = API_ERROR + + rrset = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") + self.assertFalse(rrset) def test_get_project_id(self): from certbot_dns_google.dns_google import _GoogleClient diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 5a7e07ec0..062c11650 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -190,6 +190,7 @@ class PluginsRegistry(collections.Mapping): def find_all(cls): """Find plugins using setuptools entry points.""" plugins = {} + # pylint: disable=not-callable entry_points = itertools.chain( pkg_resources.iter_entry_points( constants.SETUPTOOLS_PLUGINS_ENTRY_POINT), From e0ae356aa35adf22d154113e06dd01409df93bba Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Mar 2018 09:10:47 -0800 Subject: [PATCH 89/93] Upgrade pipstrap to 1.5.1 (#5681) * upgrade pipstrap to 1.5.1 * build leauto --- letsencrypt-auto-source/letsencrypt-auto | 38 +++++++++------------- letsencrypt-auto-source/pieces/pipstrap.py | 38 +++++++++------------- 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9ff1c1386..f97dc078d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1216,7 +1216,7 @@ UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 +"""A small script that can act as a trust root for installing pip >=8 Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, @@ -1274,7 +1274,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -1287,14 +1287,11 @@ maybe_argparse = ( if version_info < (2, 7, 0) else []) -# Pip has no dependencies, as it vendors everything: -PIP_PACKAGE = [ +PACKAGES = maybe_argparse + [ + # Pip has no dependencies, as it vendors everything: ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' 'pip-{0}.tar.gz'.format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] - - -OTHER_PACKAGES = maybe_argparse + [ + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' 'setuptools-29.0.1.tar.gz', @@ -1379,21 +1376,16 @@ def main(): index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - # We download and install pip first, then the rest, to avoid the bug - # https://github.com/certbot/certbot/issues/4938. - pip_downloads, other_downloads = [ - [hashed_download(index_base + '/packages/' + path, - temp, - digest) - for path, digest in packages] - for packages in (PIP_PACKAGE, OTHER_PACKAGES)] - for downloads in (pip_downloads, other_downloads): - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it - # otherwise sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + downloads = [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in PACKAGES] + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it otherwise + # sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index ed55b37e9..d55d5bceb 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 +"""A small script that can act as a trust root for installing pip >=8 Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, @@ -57,7 +57,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -70,14 +70,11 @@ maybe_argparse = ( if version_info < (2, 7, 0) else []) -# Pip has no dependencies, as it vendors everything: -PIP_PACKAGE = [ +PACKAGES = maybe_argparse + [ + # Pip has no dependencies, as it vendors everything: ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' 'pip-{0}.tar.gz'.format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] - - -OTHER_PACKAGES = maybe_argparse + [ + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' 'setuptools-29.0.1.tar.gz', @@ -162,21 +159,16 @@ def main(): index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - # We download and install pip first, then the rest, to avoid the bug - # https://github.com/certbot/certbot/issues/4938. - pip_downloads, other_downloads = [ - [hashed_download(index_base + '/packages/' + path, - temp, - digest) - for path, digest in packages] - for packages in (PIP_PACKAGE, OTHER_PACKAGES)] - for downloads in (pip_downloads, other_downloads): - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it - # otherwise sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + downloads = [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in PACKAGES] + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it otherwise + # sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: From f4bac423fb794c14e426defecc492306ea53cbc4 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Wed, 7 Mar 2018 15:09:47 -0800 Subject: [PATCH 90/93] fix(acme): client._revoke sends default content_type (#5687) --- acme/acme/client.py | 3 +-- acme/acme/client_test.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index d52c82a5c..9e2478afe 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -227,8 +227,7 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes response = self._post(url, messages.Revocation( certificate=cert, - reason=rsn), - content_type=None) + reason=rsn)) if response.status_code != http_client.OK: raise errors.ClientError( 'Successful revocation must return HTTP OK status') diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index a0c27e74f..00b9e19dd 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -635,8 +635,7 @@ class ClientTest(ClientTestBase): def test_revoke(self): self.client.revoke(self.certr.body, self.rsn) self.net.post.assert_called_once_with( - self.directory[messages.Revocation], mock.ANY, content_type=None, - acme_version=1) + self.directory[messages.Revocation], mock.ANY, acme_version=1) def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) @@ -776,8 +775,7 @@ class ClientV2Test(ClientTestBase): def test_revoke(self): self.client.revoke(messages_test.CERT, self.rsn) self.net.post.assert_called_once_with( - self.directory["revokeCert"], mock.ANY, content_type=None, - acme_version=2) + self.directory["revokeCert"], mock.ANY, acme_version=2) class MockJSONDeSerializable(jose.JSONDeSerializable): From cc18da926ed0c64ffd9564bcbf8cc701f6506360 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Mar 2018 11:09:31 -0800 Subject: [PATCH 91/93] Quiet pylint (#5689) --- certbot-dns-google/certbot_dns_google/dns_google_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index afab847cf..72b8be8af 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -278,6 +278,7 @@ class GoogleClientTest(unittest.TestCase): def test_get_existing_fallback(self, unused_credential_mock): client, unused_changes = self._setUp_client_with_mock( [{'managedZones': [{'id': self.zone}]}]) + # pylint: disable=no-member mock_execute = client.dns.resourceRecordSets.return_value.list.return_value.execute mock_execute.side_effect = API_ERROR From cc24b4e40af5841c8dfddeecd9bde7d1acce62e8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Mar 2018 11:12:33 -0800 Subject: [PATCH 92/93] Fix --allow-subset-of-names (#5690) * Remove aauthzr instance variable * If domain begins with fail, fail the challenge. * test --allow-subset-of-names * Fix renewal and add extra check * test after hook checks --- certbot/auth_handler.py | 80 ++++++++++++++++-------------- certbot/tests/auth_handler_test.py | 54 ++++++++++---------- tests/boulder-integration.sh | 13 +++++ tests/manual-dns-auth.sh | 12 +++-- tests/manual-dns-cleanup.sh | 11 ++-- 5 files changed, 99 insertions(+), 71 deletions(-) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 67d36c8cc..51cdf09ee 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -34,8 +34,6 @@ class AuthHandler(object): :ivar account: Client's Account :type account: :class:`certbot.account.Account` - :ivar aauthzrs: ACME Authorization Resources and their active challenges - :type aauthzrs: `list` of `AnnotatedAuthzr` :ivar list pref_challs: sorted user specified preferred challenges type strings with the most preferred challenge listed first @@ -45,7 +43,6 @@ class AuthHandler(object): self.acme = acme self.account = account - self.aauthzrs = [] self.pref_challs = pref_challs def handle_authorizations(self, orderr, best_effort=False): @@ -63,29 +60,29 @@ class AuthHandler(object): authorizations """ - for authzr in orderr.authorizations: - self.aauthzrs.append(AnnotatedAuthzr(authzr, [])) + aauthzrs = [AnnotatedAuthzr(authzr, []) + for authzr in orderr.authorizations] - self._choose_challenges() + self._choose_challenges(aauthzrs) config = zope.component.getUtility(interfaces.IConfig) notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... - while self._has_challenges(): - resp = self._solve_challenges() + while self._has_challenges(aauthzrs): + resp = self._solve_challenges(aauthzrs) logger.info("Waiting for verification...") if config.debug_challenges: notify('Challenges loaded. Press continue to submit to CA. ' 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp, best_effort) + self._respond(aauthzrs, resp, best_effort) # Just make sure all decisions are complete. - self.verify_authzr_complete() + self.verify_authzr_complete(aauthzrs) # Only return valid authorizations - retVal = [aauthzr.authzr for aauthzr in self.aauthzrs + retVal = [aauthzr.authzr for aauthzr in aauthzrs if aauthzr.authzr.body.status == messages.STATUS_VALID] if not retVal: @@ -94,10 +91,10 @@ class AuthHandler(object): return retVal - def _choose_challenges(self): + def _choose_challenges(self, aauthzrs): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: aauthzr_challenges = aauthzr.authzr.body.challenges if self.acme.acme_version == 1: combinations = aauthzr.authzr.body.combinations @@ -113,15 +110,15 @@ class AuthHandler(object): aauthzr.authzr, path) aauthzr.achalls.extend(aauthzr_achalls) - def _has_challenges(self): + def _has_challenges(self, aauthzrs): """Do we have any challenges to perform?""" - return any(aauthzr.achalls for aauthzr in self.aauthzrs) + return any(aauthzr.achalls for aauthzr in aauthzrs) - def _solve_challenges(self): + def _solve_challenges(self, aauthzrs): """Get Responses for challenges from authenticators.""" resp = [] - all_achalls = self._get_all_achalls() - with error_handler.ErrorHandler(self._cleanup_challenges): + all_achalls = self._get_all_achalls(aauthzrs) + with error_handler.ErrorHandler(self._cleanup_challenges, all_achalls): try: if all_achalls: resp = self.auth.perform(all_achalls) @@ -134,15 +131,15 @@ class AuthHandler(object): return resp - def _get_all_achalls(self): + def _get_all_achalls(self, aauthzrs): """Return all active challenges.""" all_achalls = [] - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: all_achalls.extend(aauthzr.achalls) return all_achalls - def _respond(self, resp, best_effort): + def _respond(self, aauthzrs, resp, best_effort): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -150,24 +147,27 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 chall_update = dict() - active_achalls = self._send_responses(resp, chall_update) + active_achalls = self._send_responses(aauthzrs, resp, chall_update) # Check for updated status... try: - self._poll_challenges(chall_update, best_effort) + self._poll_challenges(aauthzrs, chall_update, best_effort) finally: - self._cleanup_challenges(active_achalls) + self._cleanup_challenges(aauthzrs, active_achalls) - def _send_responses(self, resps, chall_update): + def _send_responses(self, aauthzrs, resps, chall_update): """Send responses and make sure errors are handled. + :param aauthzrs: authorizations and the selected annotated challenges + to try and perform + :type aauthzrs: `list` of `AnnotatedAuthzr` :param dict chall_update: parameter that is updated to hold aauthzr index to list of outstanding solved annotated challenges """ active_achalls = [] resps_iter = iter(resps) - for i, aauthzr in enumerate(self.aauthzrs): + for i, aauthzr in enumerate(aauthzrs): for achall in aauthzr.achalls: # This line needs to be outside of the if block below to # ensure failed challenges are cleaned up correctly @@ -184,8 +184,8 @@ class AuthHandler(object): return active_achalls - def _poll_challenges( - self, chall_update, best_effort, min_sleep=3, max_rounds=15): + def _poll_challenges(self, aauthzrs, chall_update, + best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" indices_to_check = set(chall_update.keys()) comp_indices = set() @@ -197,7 +197,7 @@ class AuthHandler(object): all_failed_achalls = set() for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( - index, chall_update[index]) + aauthzrs, index, chall_update[index]) if len(comp_achalls) == len(chall_update[index]): comp_indices.add(index) @@ -210,7 +210,7 @@ class AuthHandler(object): comp_indices.add(index) logger.warning( "Challenge failed for domain %s", - self.aauthzrs[index].authzr.body.identifier.value) + aauthzrs[index].authzr.body.identifier.value) else: all_failed_achalls.update( updated for _, updated in failed_achalls) @@ -223,14 +223,14 @@ class AuthHandler(object): comp_indices.clear() rounds += 1 - def _handle_check(self, index, achalls): + def _handle_check(self, aauthzrs, index, achalls): """Returns tuple of ('completed', 'failed').""" completed = [] failed = [] - original_aauthzr = self.aauthzrs[index] + original_aauthzr = aauthzrs[index] updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) - self.aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) + aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) if updated_authzr.body.status == messages.STATUS_VALID: return achalls, [] @@ -287,7 +287,7 @@ class AuthHandler(object): chall_prefs.extend(plugin_pref) return chall_prefs - def _cleanup_challenges(self, achall_list=None): + def _cleanup_challenges(self, aauthzrs, achall_list=None): """Cleanup challenges. If achall_list is not provided, cleanup all achallenges. @@ -296,26 +296,30 @@ class AuthHandler(object): logger.info("Cleaning up challenges") if achall_list is None: - achalls = self._get_all_achalls() + achalls = self._get_all_achalls(aauthzrs) else: achalls = achall_list if achalls: self.auth.cleanup(achalls) for achall in achalls: - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: if achall in aauthzr.achalls: aauthzr.achalls.remove(achall) break - def verify_authzr_complete(self): + def verify_authzr_complete(self, aauthzrs): """Verifies that all authorizations have been decided. + :param aauthzrs: authorizations and their selected annotated + challenges + :type aauthzrs: `list` of `AnnotatedAuthzr` + :returns: Whether all authzr are complete :rtype: bool """ - for aauthzr in self.aauthzrs: + for aauthzr in aauthzrs: authzr = aauthzr.authzr if (authzr.body.status != messages.STATUS_VALID and authzr.body.status != messages.STATUS_INVALID): diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index b6af3d0f5..54e284d9e 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -101,7 +101,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) @@ -132,7 +132,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_net.answer_challenge.call_count, 3) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) @@ -158,7 +158,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) @@ -187,7 +187,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # Check poll call self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(len(list(six.iterkeys(chall_update))), 3) self.assertTrue(0 in list(six.iterkeys(chall_update))) self.assertEqual(len(chall_update[0]), 1) @@ -278,8 +278,8 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1, unused_2): - for i, aauthzr in enumerate(self.handler.aauthzrs): + def _validate_all(self, aauthzrs, unused_1, unused_2): + for i, aauthzr in enumerate(aauthzrs): azr = aauthzr.authzr updated_azr = acme_util.gen_authzr( messages.STATUS_VALID, @@ -287,7 +287,7 @@ class HandleAuthorizationsTest(unittest.TestCase): [challb.chall for challb in azr.body.challenges], [messages.STATUS_VALID] * len(azr.body.challenges), azr.body.combinations) - self.handler.aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) + aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) class PollChallengesTest(unittest.TestCase): @@ -304,19 +304,21 @@ class PollChallengesTest(unittest.TestCase): None, self.mock_net, mock.Mock(key="mock_key"), []) self.doms = ["0", "1", "2"] - self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[0], - [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False), [])) - self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) - self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) + self.aauthzrs = [ + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[0], + [acme_util.HTTP01, acme_util.TLSSNI01], + [messages.STATUS_PENDING] * 2, False), []), + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[1], + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []), + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[2], + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []) + ] self.chall_update = {} - for i, aauthzr in enumerate(self.handler.aauthzrs): + for i, aauthzr in enumerate(self.aauthzrs): self.chall_update[i] = [ challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) for challb in aauthzr.authzr.body.challenges] @@ -324,17 +326,17 @@ class PollChallengesTest(unittest.TestCase): @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.handler._poll_challenges(self.chall_update, False) + self.handler._poll_challenges(self.aauthzrs, self.chall_update, False) - for aauthzr in self.handler.aauthzrs: + for aauthzr in self.aauthzrs: self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) @mock.patch("certbot.auth_handler.time") def test_poll_challenges_failure_best_effort(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid - self.handler._poll_challenges(self.chall_update, True) + self.handler._poll_challenges(self.aauthzrs, self.chall_update, True) - for aauthzr in self.handler.aauthzrs: + for aauthzr in self.aauthzrs: self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) @mock.patch("certbot.auth_handler.time") @@ -343,7 +345,7 @@ class PollChallengesTest(unittest.TestCase): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, - self.chall_update, False) + self.aauthzrs, self.chall_update, False) @mock.patch("certbot.auth_handler.time") def test_unable_to_find_challenge_status(self, unused_mock_time): @@ -353,11 +355,11 @@ class PollChallengesTest(unittest.TestCase): challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, - self.chall_update, False) + self.aauthzrs, self.chall_update, False) def test_verify_authzr_failure(self): - self.assertRaises( - errors.AuthorizationError, self.handler.verify_authzr_complete) + self.assertRaises(errors.AuthorizationError, + self.handler.verify_authzr_complete, self.aauthzrs) def _mock_poll_solve_one_valid(self, authzr): # Pending here because my dummy script won't change the full status. diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 2b92476fd..9748befa3 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -327,6 +327,19 @@ CheckDirHooks 1 common renew --cert-name le2.wtf CheckDirHooks 1 +# manual-dns-auth.sh will skip completing the challenge for domains that begin +# with fail. +common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ + --allow-subset-of-names \ + --preferred-challenges dns,tls-sni \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh + +if common certificates | grep "fail\.dns1\.le\.wtf"; then + echo "certificate should not have been issued for domain!" >&2 + exit 1 +fi + # ECDSA openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ diff --git a/tests/manual-dns-auth.sh b/tests/manual-dns-auth.sh index 9b9a1a5eb..febecf455 100755 --- a/tests/manual-dns-auth.sh +++ b/tests/manual-dns-auth.sh @@ -1,4 +1,8 @@ -#!/bin/sh -curl -X POST 'http://localhost:8055/set-txt' -d \ - "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\", \ - \"value\": \"$CERTBOT_VALIDATION\"}" +#!/bin/bash + +# If domain begins with fail, fail the challenge by not completing it. +if [[ "$CERTBOT_DOMAIN" != fail* ]]; then + curl -X POST 'http://localhost:8055/set-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\", \ + \"value\": \"$CERTBOT_VALIDATION\"}" +fi diff --git a/tests/manual-dns-cleanup.sh b/tests/manual-dns-cleanup.sh index 0c5c56b17..1c09e892c 100755 --- a/tests/manual-dns-cleanup.sh +++ b/tests/manual-dns-cleanup.sh @@ -1,3 +1,8 @@ -#!/bin/sh -curl -X POST 'http://localhost:8055/clear-txt' -d \ - "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}" +#!/bin/bash + +# If domain begins with fail, we didn't complete the challenge so there is +# nothing to clean up. +if [[ "$CERTBOT_DOMAIN" != fail* ]]; then + curl -X POST 'http://localhost:8055/clear-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}" +fi From 2e6d65d9ecb5f2416413597d74c3599b470a5bd4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Mar 2018 17:24:30 -0800 Subject: [PATCH 93/93] Add readthedocs requirements files (#5696) * Add readthedocs requirements files. * Only install docs extras for plugin. --- .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ .../readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-google/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-luadns/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-nsone/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-rfc2136/readthedocs.org.requirements.txt | 12 ++++++++++++ certbot-dns-route53/readthedocs.org.requirements.txt | 12 ++++++++++++ 10 files changed, 120 insertions(+) create mode 100644 certbot-dns-cloudflare/readthedocs.org.requirements.txt create mode 100644 certbot-dns-cloudxns/readthedocs.org.requirements.txt create mode 100644 certbot-dns-digitalocean/readthedocs.org.requirements.txt create mode 100644 certbot-dns-dnsimple/readthedocs.org.requirements.txt create mode 100644 certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt create mode 100644 certbot-dns-google/readthedocs.org.requirements.txt create mode 100644 certbot-dns-luadns/readthedocs.org.requirements.txt create mode 100644 certbot-dns-nsone/readthedocs.org.requirements.txt create mode 100644 certbot-dns-rfc2136/readthedocs.org.requirements.txt create mode 100644 certbot-dns-route53/readthedocs.org.requirements.txt diff --git a/certbot-dns-cloudflare/readthedocs.org.requirements.txt b/certbot-dns-cloudflare/readthedocs.org.requirements.txt new file mode 100644 index 000000000..b18901111 --- /dev/null +++ b/certbot-dns-cloudflare/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-cloudflare[docs] diff --git a/certbot-dns-cloudxns/readthedocs.org.requirements.txt b/certbot-dns-cloudxns/readthedocs.org.requirements.txt new file mode 100644 index 000000000..ae2ff8165 --- /dev/null +++ b/certbot-dns-cloudxns/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-cloudxns[docs] diff --git a/certbot-dns-digitalocean/readthedocs.org.requirements.txt b/certbot-dns-digitalocean/readthedocs.org.requirements.txt new file mode 100644 index 000000000..08d973ab3 --- /dev/null +++ b/certbot-dns-digitalocean/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-digitalocean[docs] diff --git a/certbot-dns-dnsimple/readthedocs.org.requirements.txt b/certbot-dns-dnsimple/readthedocs.org.requirements.txt new file mode 100644 index 000000000..fef73916c --- /dev/null +++ b/certbot-dns-dnsimple/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-dnsimple[docs] diff --git a/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt b/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt new file mode 100644 index 000000000..8f8c6c731 --- /dev/null +++ b/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-dnsmadeeasy[docs] diff --git a/certbot-dns-google/readthedocs.org.requirements.txt b/certbot-dns-google/readthedocs.org.requirements.txt new file mode 100644 index 000000000..6ea393f86 --- /dev/null +++ b/certbot-dns-google/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-google[docs] diff --git a/certbot-dns-luadns/readthedocs.org.requirements.txt b/certbot-dns-luadns/readthedocs.org.requirements.txt new file mode 100644 index 000000000..acb51e4ef --- /dev/null +++ b/certbot-dns-luadns/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-luadns[docs] diff --git a/certbot-dns-nsone/readthedocs.org.requirements.txt b/certbot-dns-nsone/readthedocs.org.requirements.txt new file mode 100644 index 000000000..dbdee4480 --- /dev/null +++ b/certbot-dns-nsone/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-nsone[docs] diff --git a/certbot-dns-rfc2136/readthedocs.org.requirements.txt b/certbot-dns-rfc2136/readthedocs.org.requirements.txt new file mode 100644 index 000000000..df89018ce --- /dev/null +++ b/certbot-dns-rfc2136/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-rfc2136[docs] diff --git a/certbot-dns-route53/readthedocs.org.requirements.txt b/certbot-dns-route53/readthedocs.org.requirements.txt new file mode 100644 index 000000000..660a90d0e --- /dev/null +++ b/certbot-dns-route53/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-route53[docs]