From 2d3159848448e47ccae438e8bc35a0257762549c Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 12 Apr 2018 15:47:39 -0700 Subject: [PATCH 01/93] Get mypy tox env running in the current setup (#5861) * get mypy tox env running in the current setup * use any python3 with mypy * pin mypy dependencies --- .gitignore | 1 + setup.py | 5 +++++ tools/dev_constraints.txt | 3 +++ tox.ini | 4 ++-- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 4dff20caf..e744a82a2 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ tests/letstest/venv/ # pytest cache .cache +.mypy_cache/ # docker files .docker diff --git a/setup.py b/setup.py index ba521ed2a..e674871a8 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,10 @@ dev_extras = [ 'wheel', ] +dev3_extras = [ + 'mypy', +] + docs_extras = [ 'repoze.sphinx.autointerface', # autodoc_member_order = 'bysource', autodoc_default_flags, and #4686 @@ -110,6 +114,7 @@ setup( install_requires=install_requires, extras_require={ 'dev': dev_extras, + 'dev3': dev3_extras, 'docs': docs_extras, }, diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index d02204215..df13cdbef 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -30,6 +30,7 @@ josepy==1.0.1 logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 +mypy==0.580 ndg-httpsclient==0.3.2 oauth2client==2.0.0 pathlib2==2.3.0 @@ -66,6 +67,8 @@ tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 twine==1.9.1 +typed-ast==1.1.0 +typing==3.6.4 uritemplate==0.6 virtualenv==15.1.0 wcwidth==0.1.7 diff --git a/tox.ini b/tox.ini index 049220bbb..dce5911ed 100644 --- a/tox.ini +++ b/tox.ini @@ -136,9 +136,9 @@ commands = pylint --reports=n --rcfile=.pylintrc {[base]source_paths} [testenv:mypy] -basepython = python3.4 +basepython = python3 commands = - {[base]pip_install} mypy + {[base]pip_install} .[dev3] {[base]install_packages} mypy --py2 --ignore-missing-imports {[base]source_paths} From c443db0618310c1f743b5d9bb0399d407212d295 Mon Sep 17 00:00:00 2001 From: mabayhan <30459678+mabayhan@users.noreply.github.com> Date: Thu, 12 Apr 2018 16:33:10 -0700 Subject: [PATCH 02/93] Update constants.py On FreeBSD or MacOS, "certbot --nginx" fails. The reason is, at these op. systems, nginx directory is different than linux. --- certbot-nginx/certbot_nginx/constants.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 3f263fea3..8422ab3cd 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -1,9 +1,14 @@ """nginx plugin constants.""" import pkg_resources +import platform - +if(platform.system() == ('FreeBSD' or 'Darwin')): + server_root_tmp = "/usr/local/etc/nginx" +else: + server_root_tmp = "/etc/nginx" + CLI_DEFAULTS = dict( - server_root="/etc/nginx", + server_root=server_root_tmp ctl="nginx", ) """CLI defaults.""" From a708504d5b6ad2a00311f163b3ce29bf5c27f51f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 12 Apr 2018 18:28:00 -0700 Subject: [PATCH 03/93] remove test metaclass (#5863) --- .../certbot_apache/tests/http_01_test.py | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 9ed4ee509..f120674c7 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -16,30 +16,9 @@ from certbot_apache.tests import util NUM_ACHALLS = 3 -class ApacheHttp01TestMeta(type): - """Generates parmeterized tests for testing perform.""" - def __new__(mcs, name, bases, class_dict): - - def _gen_test(num_achalls, minor_version): - def _test(self): - achalls = self.achalls[:num_achalls] - vhosts = self.vhosts[:num_achalls] - self.config.version = (2, minor_version) - self.common_perform_test(achalls, vhosts) - return _test - - for i in range(1, NUM_ACHALLS + 1): - for j in (2, 4): - test_name = "test_perform_{0}_{1}".format(i, j) - class_dict[test_name] = _gen_test(i, j) - return type.__new__(mcs, name, bases, class_dict) - - class ApacheHttp01Test(util.ApacheTest): """Test for certbot_apache.http_01.ApacheHttp01.""" - __metaclass__ = ApacheHttp01TestMeta - def setUp(self, *args, **kwargs): super(ApacheHttp01Test, self).setUp(*args, **kwargs) @@ -137,6 +116,31 @@ class ApacheHttp01Test(util.ApacheTest): self.config.config.http01_port = 12345 self.assertRaises(errors.PluginError, self.http.perform) + def test_perform_1_achall_22(self): + self.combinations_perform_test(num_achalls=1, minor_version=2) + + def test_perform_1_achall_24(self): + self.combinations_perform_test(num_achalls=1, minor_version=4) + + def test_perform_2_achall_22(self): + self.combinations_perform_test(num_achalls=2, minor_version=2) + + def test_perform_2_achall_24(self): + self.combinations_perform_test(num_achalls=2, minor_version=4) + + def test_perform_3_achall_22(self): + self.combinations_perform_test(num_achalls=3, minor_version=2) + + def test_perform_3_achall_24(self): + self.combinations_perform_test(num_achalls=3, minor_version=4) + + def combinations_perform_test(self, num_achalls, minor_version): + """Test perform with the given achall count and Apache version.""" + achalls = self.achalls[:num_achalls] + vhosts = self.vhosts[:num_achalls] + self.config.version = (2, minor_version) + self.common_perform_test(achalls, vhosts) + def common_perform_test(self, achalls, vhosts): """Tests perform with the given achalls.""" challenge_dir = self.http.challenge_dir From 6253acf335da6f462376e09adbeb89cbd58f90e9 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 12 Apr 2018 18:53:07 -0700 Subject: [PATCH 04/93] Get mypy running with clean output (#5864) Fixes #5849. * extract mypy flags into mypy.ini file * Get mypy running with clean output --- .../certbot_dns_digitalocean/dns_digitalocean_test.py | 5 +++-- certbot/crypto_util.py | 2 +- certbot/tests/cert_manager_test.py | 5 +++-- certbot/util.py | 2 +- mypy.ini | 3 +++ tox.ini | 2 +- 6 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 mypy.ini diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py index 3b8edce64..0e2043f50 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py @@ -50,7 +50,8 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic class DigitalOceanClientTest(unittest.TestCase): - id = 1 + + id_num = 1 record_prefix = "_acme-challenge" record_name = record_prefix + "." + DOMAIN record_content = "bar" @@ -70,7 +71,7 @@ class DigitalOceanClientTest(unittest.TestCase): domain_mock = mock.MagicMock() domain_mock.name = DOMAIN - domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id}} + domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id_num}} self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock] diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index bd4e7fcfc..756bd7565 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -13,7 +13,7 @@ import pyrfc3339 import six import zope.component from cryptography.hazmat.backends import default_backend -from cryptography import x509 +from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 98ff163cd..675b936b9 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -216,11 +216,12 @@ class CertificatesTest(BaseCertManagerTest): cert.is_test_cert = False parsed_certs = [cert] + mock_config = mock.MagicMock(certname=None, lineagename=None) + # pylint: disable=protected-access + # pylint: disable=protected-access get_report = lambda: cert_manager._report_human_readable(mock_config, parsed_certs) - mock_config = mock.MagicMock(certname=None, lineagename=None) - # pylint: disable=protected-access out = get_report() self.assertTrue("INVALID: EXPIRED" in out) diff --git a/certbot/util.py b/certbot/util.py index b3973d96b..55acd624f 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -54,7 +54,7 @@ _INITIAL_PID = os.getpid() # the dict are attempted to be cleaned up at program exit. If the # program exits before the lock is cleaned up, it is automatically # released, but the file isn't deleted. -_LOCKS = OrderedDict() +_LOCKS = OrderedDict() # type: OrderedDict[str, lock.LockFile] def run_script(params, log=logger.error): diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..aac60a2ad --- /dev/null +++ b/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +python_version = 2.7 +ignore_missing_imports = True diff --git a/tox.ini b/tox.ini index dce5911ed..f33802d98 100644 --- a/tox.ini +++ b/tox.ini @@ -140,7 +140,7 @@ basepython = python3 commands = {[base]pip_install} .[dev3] {[base]install_packages} - mypy --py2 --ignore-missing-imports {[base]source_paths} + mypy {[base]source_paths} [testenv:apacheconftest] #basepython = python2.7 From e0a5b1229f5d495bf147b048b02fae7bc2e231b2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 12 Apr 2018 19:02:40 -0700 Subject: [PATCH 05/93] help clarify version number (#5865) (Hopefully) helps make it clearer that that 22 and 24 corresponds to Apache 2.2 and 2.4. --- .../certbot_apache/tests/http_01_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index f120674c7..dc1ca34d6 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -50,7 +50,7 @@ class ApacheHttp01Test(util.ApacheTest): self.assertFalse(self.http.perform()) @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") - def test_enable_modules_22(self, mock_enmod): + def test_enable_modules_apache_2_2(self, mock_enmod): self.config.version = (2, 2) self.config.parser.modules.remove("authz_host_module") self.config.parser.modules.remove("mod_authz_host.c") @@ -59,7 +59,7 @@ class ApacheHttp01Test(util.ApacheTest): self.assertEqual(enmod_calls[0][0][0], "authz_host") @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") - def test_enable_modules_24(self, mock_enmod): + def test_enable_modules_apache_2_4(self, mock_enmod): self.config.parser.modules.remove("authz_core_module") self.config.parser.modules.remove("mod_authz_core.c") @@ -116,22 +116,22 @@ class ApacheHttp01Test(util.ApacheTest): self.config.config.http01_port = 12345 self.assertRaises(errors.PluginError, self.http.perform) - def test_perform_1_achall_22(self): + def test_perform_1_achall_apache_2_2(self): self.combinations_perform_test(num_achalls=1, minor_version=2) - def test_perform_1_achall_24(self): + def test_perform_1_achall_apache_2_4(self): self.combinations_perform_test(num_achalls=1, minor_version=4) - def test_perform_2_achall_22(self): + def test_perform_2_achall_apache_2_2(self): self.combinations_perform_test(num_achalls=2, minor_version=2) - def test_perform_2_achall_24(self): + def test_perform_2_achall_apache_2_4(self): self.combinations_perform_test(num_achalls=2, minor_version=4) - def test_perform_3_achall_22(self): + def test_perform_3_achall_apache_2_2(self): self.combinations_perform_test(num_achalls=3, minor_version=2) - def test_perform_3_achall_24(self): + def test_perform_3_achall_apache_2_4(self): self.combinations_perform_test(num_achalls=3, minor_version=4) def combinations_perform_test(self, num_achalls, minor_version): From 523cdc578d90ee65fa4dc0e74a53dd89a87152ea Mon Sep 17 00:00:00 2001 From: Axel Date: Fri, 13 Apr 2018 18:17:08 +0200 Subject: [PATCH 06/93] Add port option for rfc2136 plugin (#5844) --- certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py | 7 +++++-- .../certbot_dns_rfc2136/dns_rfc2136.py | 12 ++++++++---- .../certbot_dns_rfc2136/dns_rfc2136_test.py | 11 ++++++----- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py index 0f97869e2..12b360959 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py @@ -21,8 +21,9 @@ Credentials ----------- Use of this plugin requires a configuration file containing the target DNS -server that supports RFC 2136 Dynamic Updates, the name of the TSIG key, the -TSIG key secret itself and the algorithm used if it's different to HMAC-MD5. +server and optional port that supports RFC 2136 Dynamic Updates, the name +of the TSIG key, the TSIG key secret itself and the algorithm used if it's +different to HMAC-MD5. .. code-block:: ini :name: credentials.ini @@ -30,6 +31,8 @@ TSIG key secret itself and the algorithm used if it's different to HMAC-MD5. # Target DNS server dns_rfc2136_server = 192.0.2.1 + # Target DNS port + dns_rfc2136_port = 53 # TSIG key name dns_rfc2136_name = keyname. # TSIG key secret diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py index 127773469..b8c01cdd3 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py @@ -36,6 +36,8 @@ class Authenticator(dns_common.DNSAuthenticator): 'HMAC-SHA512': dns.tsig.HMAC_SHA512 } + PORT = 53 + description = 'Obtain certificates using a DNS TXT record (if you are using BIND for DNS).' ttl = 120 @@ -78,6 +80,7 @@ class Authenticator(dns_common.DNSAuthenticator): def _get_rfc2136_client(self): return _RFC2136Client(self.credentials.conf('server'), + int(self.credentials.conf('port') or self.PORT), self.credentials.conf('name'), self.credentials.conf('secret'), self.ALGORITHMS.get(self.credentials.conf('algorithm'), @@ -88,8 +91,9 @@ class _RFC2136Client(object): """ Encapsulates all communication with the target DNS server. """ - def __init__(self, server, key_name, key_secret, key_algorithm): + def __init__(self, server, port, key_name, key_secret, key_algorithm): self.server = server + self.port = port self.keyring = dns.tsigkeyring.from_text({ key_name: key_secret }) @@ -118,7 +122,7 @@ class _RFC2136Client(object): update.add(rel, record_ttl, dns.rdatatype.TXT, record_content) try: - response = dns.query.tcp(update, self.server) + response = dns.query.tcp(update, self.server, port=self.port) except Exception as e: raise errors.PluginError('Encountered error adding TXT record: {0}' .format(e)) @@ -153,7 +157,7 @@ class _RFC2136Client(object): update.delete(rel, dns.rdatatype.TXT, record_content) try: - response = dns.query.tcp(update, self.server) + response = dns.query.tcp(update, self.server, port=self.port) except Exception as e: raise errors.PluginError('Encountered error deleting TXT record: {0}' .format(e)) @@ -202,7 +206,7 @@ class _RFC2136Client(object): request.flags ^= dns.flags.RD try: - response = dns.query.udp(request, self.server) + response = dns.query.udp(request, self.server, port=self.port) rcode = response.rcode() # Authoritative Answer bit should be set diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py index 8a5166330..89ce3d93e 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py @@ -14,6 +14,7 @@ from certbot.plugins.dns_test_common import DOMAIN from certbot.tests import util as test_util SERVER = '192.0.2.1' +PORT = 53 NAME = 'a-tsig-key.' SECRET = 'SSB3b25kZXIgd2hvIHdpbGwgYm90aGVyIHRvIGRlY29kZSB0aGlzIHRleHQK' VALID_CONFIG = {"rfc2136_server": SERVER, "rfc2136_name": NAME, "rfc2136_secret": SECRET} @@ -74,7 +75,7 @@ class RFC2136ClientTest(unittest.TestCase): def setUp(self): from certbot_dns_rfc2136.dns_rfc2136 import _RFC2136Client - self.rfc2136_client = _RFC2136Client(SERVER, NAME, SECRET, dns.tsig.HMAC_MD5) + self.rfc2136_client = _RFC2136Client(SERVER, PORT, NAME, SECRET, dns.tsig.HMAC_MD5) @mock.patch("dns.query.tcp") def test_add_txt_record(self, query_mock): @@ -84,7 +85,7 @@ class RFC2136ClientTest(unittest.TestCase): self.rfc2136_client.add_txt_record("bar", "baz", 42) - query_mock.assert_called_with(mock.ANY, SERVER) + query_mock.assert_called_with(mock.ANY, SERVER, port=PORT) self.assertTrue("bar. 42 IN TXT \"baz\"" in str(query_mock.call_args[0][0])) @mock.patch("dns.query.tcp") @@ -117,7 +118,7 @@ class RFC2136ClientTest(unittest.TestCase): self.rfc2136_client.del_txt_record("bar", "baz") - query_mock.assert_called_with(mock.ANY, SERVER) + query_mock.assert_called_with(mock.ANY, SERVER, port=PORT) self.assertTrue("bar. 0 NONE TXT \"baz\"" in str(query_mock.call_args[0][0])) @mock.patch("dns.query.tcp") @@ -169,7 +170,7 @@ class RFC2136ClientTest(unittest.TestCase): # _query_soa | pylint: disable=protected-access result = self.rfc2136_client._query_soa(DOMAIN) - query_mock.assert_called_with(mock.ANY, SERVER) + query_mock.assert_called_with(mock.ANY, SERVER, port=PORT) self.assertTrue(result == True) @mock.patch("dns.query.udp") @@ -179,7 +180,7 @@ class RFC2136ClientTest(unittest.TestCase): # _query_soa | pylint: disable=protected-access result = self.rfc2136_client._query_soa(DOMAIN) - query_mock.assert_called_with(mock.ANY, SERVER) + query_mock.assert_called_with(mock.ANY, SERVER, port=PORT) self.assertTrue(result == False) @mock.patch("dns.query.udp") From 590ec375ec89cce2a10e984c86fbf23d4533108f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 13 Apr 2018 16:10:58 -0700 Subject: [PATCH 07/93] Get mypy running in travis for easier review (#5875) --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9ec2f724b..370137f68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,8 @@ matrix: addons: - python: "2.7" env: TOXENV=lint + - python: "3.5" + env: TOXENV=mypy - python: "2.7" env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest' sudo: required From b39507c5af53bd8511dec554a653cd7a4d3e8267 Mon Sep 17 00:00:00 2001 From: mabayhan <30459678+mabayhan@users.noreply.github.com> Date: Tue, 17 Apr 2018 09:09:27 -0700 Subject: [PATCH 08/93] Update constants.py Fixed comma. --- certbot-nginx/certbot_nginx/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 8422ab3cd..72fc5d4de 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -8,7 +8,7 @@ else: server_root_tmp = "/etc/nginx" CLI_DEFAULTS = dict( - server_root=server_root_tmp + server_root=server_root_tmp, ctl="nginx", ) """CLI defaults.""" From 6dc8b66760cc7ab120275bca4117c12f6c25c19a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 17 Apr 2018 11:27:45 -0700 Subject: [PATCH 09/93] Add _choose_lineagename and update docs (#5650) --- certbot/client.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 2992c0cec..60e1ca37e 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -338,9 +338,10 @@ class Client(object): authenticator and installer, and then create a new renewable lineage containing it. - :param list domains: Domains to request. - :param plugins: A PluginsFactory object. - :param str certname: Name of new cert + :param domains: domains to request a certificate for + :type domains: `list` of `str` + :param certname: requested name of lineage + :type certname: `str` or `None` :returns: A new :class:`certbot.storage.RenewableCert` instance referred to the enrolled cert lineage, False if the cert could not @@ -355,13 +356,7 @@ class Client(object): "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") - if certname: - new_name = certname - elif util.is_wildcard_domain(domains[0]): - # Don't make files and directories starting with *. - new_name = domains[0][2:] - else: - new_name = domains[0] + new_name = self._choose_lineagename(domains, certname) if self.config.dry_run: logger.debug("Dry run: Skipping creating new lineage for %s", @@ -373,6 +368,26 @@ class Client(object): key.pem, chain, self.config) + def _choose_lineagename(self, domains, certname): + """Chooses a name for the new lineage. + + :param domains: domains in certificate request + :type domains: `list` of `str` + :param certname: requested name of lineage + :type certname: `str` or `None` + + :returns: lineage name that should be used + :rtype: str + + """ + if certname: + return certname + elif util.is_wildcard_domain(domains[0]): + # Don't make files and directories starting with *. + return domains[0][2:] + else: + return domains[0] + def save_certificate(self, cert_pem, chain_pem, cert_path, chain_path, fullchain_path): """Saves the certificate received from the ACME server. From 5c7fc07ccfca34e4ecfc4be4d124e921fdc8440e Mon Sep 17 00:00:00 2001 From: Kiel C Date: Tue, 17 Apr 2018 13:52:39 -0700 Subject: [PATCH 10/93] Adjust file paths message from Warning to Info. (#5743) --- certbot/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/client.py b/certbot/client.py index 60e1ca37e..cf2afa2e5 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -352,7 +352,7 @@ class Client(object): if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): - logger.warning( + logger.info( "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") From a9e01ade4c2d42d48e6e9a77d975a28d7f0533b9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 17 Apr 2018 17:17:15 -0700 Subject: [PATCH 11/93] Revert "use older boulder version (#5852)" (#5855) This reverts commit 6b29d159a2f221c3437770bdb43924ee6f953c4b. --- tests/boulder-fetch.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 53485ffc0..fc9cbaae7 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -7,11 +7,10 @@ set -xe export GOPATH=${GOPATH:-$HOME/gopath} BOULDERPATH=${BOULDERPATH:-$GOPATH/src/github.com/letsencrypt/boulder} if [ ! -d ${BOULDERPATH} ]; then - git clone https://github.com/letsencrypt/boulder ${BOULDERPATH} + git clone --depth=1 https://github.com/letsencrypt/boulder ${BOULDERPATH} fi cd ${BOULDERPATH} -git checkout fa5c9176655d9fa8dfca188de08bd5373aca422f FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ifconfig docker0 | grep "inet " | xargs | cut -d ' ' -f 2) [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) From 261d063b10fdaa9a4e4d23763af0769965996a1b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 18 Apr 2018 10:02:31 -0700 Subject: [PATCH 12/93] Revert fix-macos-pytest (#5853) * Revert "Fix pytest on macOS in Travis (#5360)" This reverts commit 5388842e5b3868e29caf545fb771a23e7fce4143. * remove oldest passenv --- pytest.ini | 2 -- tools/install_and_test.sh | 2 +- tools/pytest.sh | 15 --------------- tox.cover.sh | 3 +-- tox.ini | 16 ---------------- 5 files changed, 2 insertions(+), 36 deletions(-) delete mode 100644 pytest.ini delete mode 100755 tools/pytest.sh diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b64550cb7..000000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts = --quiet diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index f0385470b..59832cbc3 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -19,5 +19,5 @@ for requirement in "$@" ; do if [ $pkg = "." ]; then pkg="certbot" fi - "$(dirname $0)/pytest.sh" --pyargs $pkg + pytest --numprocesses auto --quiet --pyargs $pkg done diff --git a/tools/pytest.sh b/tools/pytest.sh deleted file mode 100755 index 8e3619d5d..000000000 --- a/tools/pytest.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Runs pytest with the provided arguments, adding --numprocesses to the command -# line. This argument is set to "auto" if the environmnent variable TRAVIS is -# not set, otherwise, it is set to 2. This works around -# https://github.com/pytest-dev/pytest-xdist/issues/9. Currently every Travis -# environnment provides two cores. See -# https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments. - -if ${TRAVIS:-false}; then - NUMPROCESSES="2" -else - NUMPROCESSES="auto" -fi - -pytest --numprocesses "$NUMPROCESSES" "$@" diff --git a/tox.cover.sh b/tox.cover.sh index bc0e5a8bf..2b5a3cf19 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -51,8 +51,7 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest="$(dirname $0)/tools/pytest.sh" - "$pytest" --cov "$pkg_dir" --cov-append --cov-report= --pyargs "$1" + pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index f33802d98..38d4e6ae1 100644 --- a/tox.ini +++ b/tox.ini @@ -60,8 +60,6 @@ commands = setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 -passenv = - TRAVIS [testenv:py27-oldest] commands = @@ -69,40 +67,30 @@ commands = setenv = {[testenv]setenv} CERTBOT_OLDEST=1 -passenv = - {[testenv]passenv} [testenv:py27-acme-oldest] commands = {[base]install_and_test} acme[dev] setenv = {[testenv:py27-oldest]setenv} -passenv = - {[testenv:py27-oldest]passenv} [testenv:py27-apache-oldest] commands = {[base]install_and_test} certbot-apache setenv = {[testenv:py27-oldest]setenv} -passenv = - {[testenv:py27-oldest]passenv} [testenv:py27-certbot-oldest] commands = {[base]install_and_test} .[dev] setenv = {[testenv:py27-oldest]setenv} -passenv = - {[testenv:py27-oldest]passenv} [testenv:py27-dns-oldest] commands = {[base]install_and_test} {[base]dns_packages} setenv = {[testenv:py27-oldest]setenv} -passenv = - {[testenv:py27-oldest]passenv} [testenv:py27-nginx-oldest] commands = @@ -110,8 +98,6 @@ commands = python tests/lock_test.py setenv = {[testenv:py27-oldest]setenv} -passenv = - {[testenv:py27-oldest]passenv} [testenv:py27_install] basepython = python2.7 @@ -123,8 +109,6 @@ basepython = python2.7 commands = {[base]install_packages} ./tox.cover.sh -passenv = - {[testenv]passenv} [testenv:lint] basepython = python2.7 From a024aaf59d04e7f62b2a8e3d6a67f4db75c76345 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 18 Apr 2018 21:08:30 +0300 Subject: [PATCH 13/93] Enhance verb (#5596) * Add the cli parameters * Tests and error messages * Requested fixes * Only handle SSL vhosts * Interactive cert-name selection if not defined on CLI * Address PR comments * Address review comments * Added tests and addressed review comments * Move cert manager tests to correct file * Add display ops tests * Use display util constants instead of hardcoded values in tests --- certbot-apache/certbot_apache/configurator.py | 36 ++++-- .../certbot_apache/tests/configurator_test.py | 38 +++++- .../certbot_apache/tests/debian_test.py | 4 + certbot-apache/certbot_apache/tls_sni_01.py | 3 +- certbot/cert_manager.py | 32 ++--- certbot/cli.py | 28 +++-- certbot/client.py | 15 ++- certbot/display/ops.py | 32 ++++- certbot/main.py | 53 ++++++++- certbot/plugins/selection.py | 14 ++- certbot/tests/cert_manager_test.py | 98 ++++++++++++++++ certbot/tests/client_test.py | 23 ++++ certbot/tests/display/ops_test.py | 50 +++++++- certbot/tests/main_test.py | 109 ++++++++++++++++++ 14 files changed, 480 insertions(+), 55 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 722e94e18..03ba05bb0 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -323,7 +323,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Returned objects are guaranteed to be ssl vhosts return self._choose_vhosts_wildcard(domain, create_if_no_ssl) else: - return [self.choose_vhost(domain)] + return [self.choose_vhost(domain, create_if_no_ssl)] def _vhosts_for_wildcard(self, domain): """ @@ -475,20 +475,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if chain_path is not None: self.save_notes += "\tSSLCertificateChainFile %s\n" % chain_path - def choose_vhost(self, target_name, temp=False): + def choose_vhost(self, target_name, create_if_no_ssl=True): """Chooses a virtual host based on the given domain name. If there is no clear virtual host to be selected, the user is prompted with all available choices. - The returned vhost is guaranteed to have TLS enabled unless temp is - True. If temp is True, there is no such guarantee and the result is - not cached. + The returned vhost is guaranteed to have TLS enabled unless + create_if_no_ssl is set to False, in which case there is no such guarantee + and the result is not cached. :param str target_name: domain name - :param bool temp: whether the vhost is only used temporarily + :param bool create_if_no_ssl: If found VirtualHost doesn't have a HTTPS + counterpart, should one get created - :returns: ssl vhost associated with name + :returns: vhost associated with name :rtype: :class:`~certbot_apache.obj.VirtualHost` :raises .errors.PluginError: If no vhost is available or chosen @@ -501,7 +502,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Try to find a reasonable vhost vhost = self._find_best_vhost(target_name) if vhost is not None: - if temp: + if not create_if_no_ssl: return vhost if not vhost.ssl: vhost = self.make_vhost_ssl(vhost) @@ -510,7 +511,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc[target_name] = vhost return vhost - return self._choose_vhost_from_list(target_name, temp) + # Negate create_if_no_ssl value to indicate if we want a SSL vhost + # to get created if a non-ssl vhost is selected. + return self._choose_vhost_from_list(target_name, temp=not create_if_no_ssl) def _choose_vhost_from_list(self, target_name, temp=False): # Select a vhost from a list @@ -1505,7 +1508,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) - vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) + matched_vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) + # We should be handling only SSL vhosts for enhancements + vhosts = [vhost for vhost in matched_vhosts if vhost.ssl] + + if not vhosts: + msg_tmpl = ("Certbot was not able to find SSL VirtualHost for a " + "domain {0} for enabling enhancement \"{1}\". The requested " + "enhancement was not configured.") + msg_enhancement = enhancement + if options: + msg_enhancement += ": " + options + msg = msg_tmpl.format(domain, msg_enhancement) + logger.warning(msg) + raise errors.PluginError(msg) try: for vhost in vhosts: func(vhost, options) diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 7b1e4fa86..e33e16843 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -246,7 +246,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache.display_ops.select_vhost") def test_choose_vhost_select_vhost_with_temp(self, mock_select): mock_select.return_value = self.vh_truth[0] - chosen_vhost = self.config.choose_vhost("none.com", temp=True) + chosen_vhost = self.config.choose_vhost("none.com", create_if_no_ssl=False) self.assertEqual(self.vh_truth[0], chosen_vhost) @mock.patch("certbot_apache.display_ops.select_vhost") @@ -936,6 +936,22 @@ class MultipleVhostsTest(util.ApacheTest): errors.PluginError, self.config.enhance, "certbot.demo", "unknown_enhancement") + def test_enhance_no_ssl_vhost(self): + with mock.patch("certbot_apache.configurator.logger.warning") as mock_log: + self.assertRaises(errors.PluginError, self.config.enhance, + "certbot.demo", "redirect") + # Check that correct logger.warning was printed + self.assertTrue("not able to find" in mock_log.call_args[0][0]) + self.assertTrue("\"redirect\"" in mock_log.call_args[0][0]) + + mock_log.reset_mock() + + self.assertRaises(errors.PluginError, self.config.enhance, + "certbot.demo", "ensure-http-header", "Test") + # Check that correct logger.warning was printed + self.assertTrue("not able to find" in mock_log.call_args[0][0]) + self.assertTrue("Test" in mock_log.call_args[0][0]) + @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling(self, mock_exe): self.config.parser.update_runtime_variables = mock.Mock() @@ -945,6 +961,7 @@ class MultipleVhostsTest(util.ApacheTest): mock_exe.return_value = True # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "staple-ocsp") # Get the ssl vhost for certbot.demo @@ -971,6 +988,7 @@ class MultipleVhostsTest(util.ApacheTest): mock_exe.return_value = True # Checking the case with already enabled ocsp stapling configuration + self.config.choose_vhost("ocspvhost.com") self.config.enhance("ocspvhost.com", "staple-ocsp") # Get the ssl vhost for letsencrypt.demo @@ -995,6 +1013,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.parser.modules.add("mod_ssl.c") self.config.parser.modules.add("socache_shmcb_module") self.config.get_version = mock.Mock(return_value=(2, 2, 0)) + self.config.choose_vhost("certbot.demo") self.assertRaises(errors.PluginError, self.config.enhance, "certbot.demo", "staple-ocsp") @@ -1020,6 +1039,7 @@ class MultipleVhostsTest(util.ApacheTest): mock_exe.return_value = True # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "ensure-http-header", "Strict-Transport-Security") @@ -1039,7 +1059,8 @@ class MultipleVhostsTest(util.ApacheTest): # skip the enable mod self.config.parser.modules.add("headers_module") - # This will create an ssl vhost for certbot.demo + # This will create an ssl vhost for encryption-example.demo + self.config.choose_vhost("encryption-example.demo") self.config.enhance("encryption-example.demo", "ensure-http-header", "Strict-Transport-Security") @@ -1058,6 +1079,7 @@ class MultipleVhostsTest(util.ApacheTest): mock_exe.return_value = True # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "ensure-http-header", "Upgrade-Insecure-Requests") @@ -1079,7 +1101,8 @@ class MultipleVhostsTest(util.ApacheTest): # skip the enable mod self.config.parser.modules.add("headers_module") - # This will create an ssl vhost for certbot.demo + # This will create an ssl vhost for encryption-example.demo + self.config.choose_vhost("encryption-example.demo") self.config.enhance("encryption-example.demo", "ensure-http-header", "Upgrade-Insecure-Requests") @@ -1097,6 +1120,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.get_version = mock.Mock(return_value=(2, 2)) # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "redirect") # These are not immediately available in find_dir even with save() and @@ -1147,6 +1171,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.save() # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "redirect") # These are not immediately available in find_dir even with save() and @@ -1213,6 +1238,9 @@ class MultipleVhostsTest(util.ApacheTest): self.config.parser.modules.add("rewrite_module") self.config.get_version = mock.Mock(return_value=(2, 3, 9)) + # Creates ssl vhost for the domain + self.config.choose_vhost("red.blue.purple.com") + self.config.enhance("red.blue.purple.com", "redirect") verify_no_redirect = ("certbot_apache.configurator." "ApacheConfigurator._verify_no_certbot_redirect") @@ -1224,7 +1252,7 @@ class MultipleVhostsTest(util.ApacheTest): # Skip the enable mod self.config.parser.modules.add("rewrite_module") self.config.get_version = mock.Mock(return_value=(2, 3, 9)) - + self.config.choose_vhost("red.blue.purple.com") self.config.enhance("red.blue.purple.com", "redirect") # Clear state about enabling redirect on this run # pylint: disable=protected-access @@ -1446,6 +1474,7 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.config.parser.modules.add("mod_ssl.c") self.config.parser.modules.add("headers_module") + self.vh_truth[3].ssl = True self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] self.config.enhance("*.certbot.demo", "ensure-http-header", "Upgrade-Insecure-Requests") @@ -1453,6 +1482,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") def test_enhance_wildcard_no_install(self, mock_choose): + self.vh_truth[3].ssl = True mock_choose.return_value = [self.vh_truth[3]] self.config.parser.modules.add("mod_ssl.c") self.config.parser.modules.add("headers_module") diff --git a/certbot-apache/certbot_apache/tests/debian_test.py b/certbot-apache/certbot_apache/tests/debian_test.py index a648101e9..fde8d4c35 100644 --- a/certbot-apache/certbot_apache/tests/debian_test.py +++ b/certbot-apache/certbot_apache/tests/debian_test.py @@ -161,6 +161,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config.parser.modules.add("mod_ssl.c") self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True + # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "staple-ocsp") self.assertTrue("socache_shmcb_module" in self.config.parser.modules) @@ -172,6 +174,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): mock_exe.return_value = True # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "ensure-http-header", "Strict-Transport-Security") self.assertTrue("headers_module" in self.config.parser.modules) @@ -183,6 +186,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2)) # This will create an ssl vhost for certbot.demo + self.config.choose_vhost("certbot.demo") self.config.enhance("certbot.demo", "redirect") self.assertTrue("rewrite_module" in self.config.parser.modules) diff --git a/certbot-apache/certbot_apache/tls_sni_01.py b/certbot-apache/certbot_apache/tls_sni_01.py index 5ce96ac5f..549feb17d 100644 --- a/certbot-apache/certbot_apache/tls_sni_01.py +++ b/certbot-apache/certbot_apache/tls_sni_01.py @@ -123,7 +123,8 @@ class ApacheTlsSni01(common.TLSSNI01): self.configurator.config.tls_sni_01_port))) try: - vhost = self.configurator.choose_vhost(achall.domain, temp=True) + vhost = self.configurator.choose_vhost(achall.domain, + create_if_no_ssl=False) except (PluginError, MissingCommandlineFlag): # We couldn't find the virtualhost for this domain, possibly # because it's a new vhost that's not configured yet diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index 4240a0523..d841c1912 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -46,7 +46,7 @@ def rename_lineage(config): """ disp = zope.component.getUtility(interfaces.IDisplay) - certname = _get_certnames(config, "rename")[0] + certname = get_certnames(config, "rename")[0] new_certname = config.new_certname if not new_certname: @@ -88,7 +88,7 @@ def certificates(config): def delete(config): """Delete Certbot files associated with a certificate lineage.""" - certnames = _get_certnames(config, "delete", allow_multiple=True) + certnames = get_certnames(config, "delete", allow_multiple=True) for certname in certnames: storage.delete_files(config, certname) disp = zope.component.getUtility(interfaces.IDisplay) @@ -288,11 +288,7 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False): cert.privkey)) return "".join(certinfo) -################### -# Private Helpers -################### - -def _get_certnames(config, verb, allow_multiple=False): +def get_certnames(config, verb, allow_multiple=False, custom_prompt=None): """Get certname from flag, interactively, or error out. """ certname = config.certname @@ -305,22 +301,32 @@ def _get_certnames(config, verb, allow_multiple=False): if not choices: raise errors.Error("No existing certificates found.") if allow_multiple: + if not custom_prompt: + prompt = "Which certificate(s) would you like to {0}?".format(verb) + else: + prompt = custom_prompt code, certnames = disp.checklist( - "Which certificate(s) would you like to {0}?".format(verb), - choices, cli_flag="--cert-name", - force_interactive=True) + prompt, choices, cli_flag="--cert-name", force_interactive=True) if code != display_util.OK: raise errors.Error("User ended interaction.") else: - code, index = disp.menu("Which certificate would you like to {0}?".format(verb), - choices, cli_flag="--cert-name", - force_interactive=True) + if not custom_prompt: + prompt = "Which certificate would you like to {0}?".format(verb) + else: + prompt = custom_prompt + + code, index = disp.menu( + prompt, choices, cli_flag="--cert-name", force_interactive=True) if code != display_util.OK or index not in range(0, len(choices)): raise errors.Error("User ended interaction.") certnames = [choices[index]] return certnames +################### +# Private Helpers +################### + def _report_lines(msgs): """Format a results report for a category of single-line renewal outcomes""" return " " + "\n ".join(str(msg) for msg in msgs) diff --git a/certbot/cli.py b/certbot/cli.py index 1c2273c8a..9584c3904 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -76,6 +76,7 @@ obtain, install, and renew certificates: (default) run Obtain & install a certificate in your current webserver certonly Obtain or renew a certificate, but do not install it renew Renew all previously obtained certificates that are near expiry + enhance Add security enhancements to your existing configuration -d DOMAINS Comma-separated list of domains to obtain a certificate for %s @@ -415,6 +416,12 @@ VERB_HELP = [ os.path.join(flag_default("config_dir"), "live"))), "usage": "\n\n certbot update_symlinks [options]\n\n" }), + ("enhance", { + "short": "Add security enhancements to your existing configuration", + "opts": ("Helps to harden the TLS configration by adding security enhancements " + "to already existing configuration."), + "usage": "\n\n certbot enhance [options]\n\n" + }), ] # VERB_HELP is a list in order to preserve order, but a dict is sometimes useful @@ -449,6 +456,7 @@ class HelpfulArgumentParser(object): "update_symlinks": main.update_symlinks, "certificates": main.certificates, "delete": main.delete, + "enhance": main.enhance, } # Get notification function for printing @@ -883,21 +891,22 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "flag to 0 disables log rotation entirely, causing " "Certbot to always append to the same log file.") helpful.add( - [None, "automation", "run", "certonly"], "-n", "--non-interactive", "--noninteractive", + [None, "automation", "run", "certonly", "enhance"], + "-n", "--non-interactive", "--noninteractive", dest="noninteractive_mode", action="store_true", default=flag_default("noninteractive_mode"), help="Run without ever asking for user input. This may require " "additional command line flags; the client will try to explain " "which ones are required if it finds one missing") helpful.add( - [None, "register", "run", "certonly"], + [None, "register", "run", "certonly", "enhance"], constants.FORCE_INTERACTIVE_FLAG, action="store_true", default=flag_default("force_interactive"), help="Force Certbot to be interactive even if it detects it's not " "being run in a terminal. This flag cannot be used with the " "renew subcommand.") helpful.add( - [None, "run", "certonly", "certificates"], + [None, "run", "certonly", "certificates", "enhance"], "-d", "--domains", "--domain", dest="domains", metavar="DOMAIN", action=_DomainsAction, default=flag_default("domains"), @@ -913,8 +922,8 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "name. In the case of a name collision it will append a number " "like 0001 to the file path name. (default: Ask)") helpful.add( - [None, "run", "certonly", "manage", "delete", "certificates", "renew"], - "--cert-name", dest="certname", + [None, "run", "certonly", "manage", "delete", "certificates", + "renew", "enhance"], "--cert-name", dest="certname", metavar="CERTNAME", default=flag_default("certname"), help="Certificate name to apply. This name is used by Certbot for housekeeping " "and in file paths; it doesn't affect the content of the certificate itself. " @@ -1085,7 +1094,8 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis dest="must_staple", default=flag_default("must_staple"), help=config_help("must_staple")) helpful.add( - "security", "--redirect", action="store_true", dest="redirect", + ["security", "enhance"], + "--redirect", action="store_true", dest="redirect", default=flag_default("redirect"), help="Automatically redirect all HTTP traffic to HTTPS for the newly " "authenticated vhost. (default: Ask)") @@ -1095,7 +1105,8 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis help="Do not automatically redirect all HTTP traffic to HTTPS for the newly " "authenticated vhost. (default: Ask)") helpful.add( - "security", "--hsts", action="store_true", dest="hsts", default=flag_default("hsts"), + ["security", "enhance"], + "--hsts", action="store_true", dest="hsts", default=flag_default("hsts"), help="Add the Strict-Transport-Security header to every HTTP response." " Forcing browser to always use SSL for the domain." " Defends against SSL Stripping.") @@ -1103,7 +1114,8 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "security", "--no-hsts", action="store_false", dest="hsts", default=flag_default("hsts"), help=argparse.SUPPRESS) helpful.add( - "security", "--uir", action="store_true", dest="uir", default=flag_default("uir"), + ["security", "enhance"], + "--uir", action="store_true", dest="uir", default=flag_default("uir"), help='Add the "Content-Security-Policy: upgrade-insecure-requests"' ' header to every HTTP response. Forcing the browser to use' ' https:// for every http:// resource.') diff --git a/certbot/client.py b/certbot/client.py index cf2afa2e5..45dc9c63b 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -466,7 +466,7 @@ class Client(object): # sites may have been enabled / final cleanup self.installer.restart() - def enhance_config(self, domains, chain_path): + def enhance_config(self, domains, chain_path, ask_redirect=True): """Enhance the configuration. :param list domains: list of domains to configure @@ -493,8 +493,9 @@ class Client(object): for config_name, enhancement_name, option in enhancement_info: config_value = getattr(self.config, config_name) if enhancement_name in supported: - if config_name == "redirect" and config_value is None: - config_value = enhancements.ask(enhancement_name) + if ask_redirect: + if config_name == "redirect" and config_value is None: + config_value = enhancements.ask(enhancement_name) if config_value: self.apply_enhancement(domains, enhancement_name, option) enhanced = True @@ -530,8 +531,12 @@ class Client(object): try: self.installer.enhance(dom, enhancement, options) except errors.PluginEnhancementAlreadyPresent: - logger.warning("Enhancement %s was already set.", - enhancement) + if enhancement == "ensure-http-header": + logger.warning("Enhancement %s was already set.", + options) + else: + logger.warning("Enhancement %s was already set.", + enhancement) except errors.PluginError: logger.warning("Unable to set enhancement %s for %s", enhancement, dom) diff --git a/certbot/display/ops.py b/certbot/display/ops.py index 57d2362fd..1e15a8474 100644 --- a/certbot/display/ops.py +++ b/certbot/display/ops.py @@ -86,13 +86,31 @@ def choose_account(accounts): else: return None +def choose_values(values, question=None): + """Display screen to let user pick one or multiple values from the provided + list. -def choose_names(installer): + :param list values: Values to select from + + :returns: List of selected values + :rtype: list + """ + code, items = z_util(interfaces.IDisplay).checklist( + question, tags=values, force_interactive=True) + if code == display_util.OK and items: + return items + else: + return [] + +def choose_names(installer, question=None): """Display screen to select domains to validate. :param installer: An installer object :type installer: :class:`certbot.interfaces.IInstaller` + :param `str` question: Overriding dialog question to ask the user if asked + to choose from domain names. + :returns: List of selected names :rtype: `list` of `str` @@ -108,7 +126,7 @@ def choose_names(installer): return _choose_names_manually( "No names were found in your configuration files. ") - code, names = _filter_names(names) + code, names = _filter_names(names, question) if code == display_util.OK and names: return names else: @@ -142,7 +160,7 @@ def _sort_names(FQDNs): return sorted(FQDNs, key=lambda fqdn: fqdn.split('.')[::-1][1:]) -def _filter_names(names): +def _filter_names(names, override_question=None): """Determine which names the user would like to select from a list. :param list names: domain names @@ -155,10 +173,12 @@ def _filter_names(names): """ #Sort by domain first, and then by subdomain sorted_names = _sort_names(names) - + if override_question: + question = override_question + else: + question = "Which names would you like to activate HTTPS for?" code, names = z_util(interfaces.IDisplay).checklist( - "Which names would you like to activate HTTPS for?", - tags=sorted_names, cli_flag="--domains", force_interactive=True) + question, tags=sorted_names, cli_flag="--domains", force_interactive=True) return code, [str(s) for s in names] diff --git a/certbot/main.py b/certbot/main.py index 7be852e83..dd0991c8d 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -382,7 +382,7 @@ def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): if not obj.yesno(msg, "Update cert", "Cancel", default=True): raise errors.ConfigurationError("Specified mismatched cert name and domains.") -def _find_domains_or_certname(config, installer): +def _find_domains_or_certname(config, installer, question=None): """Retrieve domains and certname from config or user input. :param config: Configuration object @@ -391,6 +391,8 @@ def _find_domains_or_certname(config, installer): :param installer: Installer object :type installer: interfaces.IInstaller + :param `str` question: Overriding dialog question to ask the user if asked + to choose from domain names. :returns: Two-part tuple of domains and certname :rtype: `tuple` of list of `str` and `str` @@ -411,7 +413,7 @@ def _find_domains_or_certname(config, installer): # that certname might not have existed, or there was a problem. # try to get domains from the user. if not domains: - domains = display_ops.choose_names(installer) + domains = display_ops.choose_names(installer, question) if not domains and not certname: raise errors.Error("Please specify --domains, or --installer that " @@ -859,6 +861,53 @@ def plugins_cmd(config, plugins): logger.debug("Prepared plugins: %s", available) notify(str(available)) +def enhance(config, plugins): + """Add security enhancements to existing configuration + + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: List of plugins + :type plugins: `list` of `str` + + :returns: `None` + :rtype: None + + """ + supported_enhancements = ["hsts", "redirect", "uir", "staple"] + # Check that at least one enhancement was requested on command line + if not any([getattr(config, enh) for enh in supported_enhancements]): + msg = ("Please specify one or more enhancement types to configure. To list " + "the available enhancement types, run:\n\n%s --help enhance\n") + logger.warning(msg, sys.argv[0]) + raise errors.MisconfigurationError("No enhancements requested, exiting.") + + try: + installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "enhance") + except errors.PluginSelectionError as e: + return str(e) + + certname_question = ("Which certificate would you like to use to enhance " + "your configuration?") + config.certname = cert_manager.get_certnames( + config, "enhance", allow_multiple=False, + custom_prompt=certname_question)[0] + cert_domains = cert_manager.domains_for_certname(config, config.certname) + if config.noninteractive_mode: + domains = cert_domains + else: + domain_question = ("Which domain names would you like to enable the " + "selected enhancements for?") + domains = display_ops.choose_values(cert_domains, domain_question) + if not domains: + raise errors.Error("User cancelled the domain selection. No domains " + "defined, exiting.") + if not config.chain_path: + lineage = cert_manager.lineage_for_certname(config, config.certname) + config.chain_path = lineage.chain_path + le_client = _init_le_client(config, authenticator=None, installer=installer) + le_client.enhance_config(domains, config.chain_path, ask_redirect=False) + def rollback(config, plugins): """Rollback server configuration changes made during install. diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 5b1953187..030d5b6db 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -147,6 +147,7 @@ def record_chosen_plugins(config, plugins, auth, inst): def choose_configurator_plugins(config, plugins, verb): + # pylint: disable=too-many-branches """ Figure out which configurator we're going to use, modifies config.authenticator and config.installer strings to reflect that choice if @@ -159,6 +160,11 @@ def choose_configurator_plugins(config, plugins, verb): """ req_auth, req_inst = cli_plugin_requests(config) + installer_question = None + + if verb == "enhance": + installer_question = ("Which installer would you like to use to " + "configure the selected enhancements?") # Which plugins do we need? if verb == "run": @@ -176,11 +182,11 @@ def choose_configurator_plugins(config, plugins, verb): need_inst = need_auth = False if verb == "certonly": need_auth = True - if verb == "install": + if verb == "install" or verb == "enhance": need_inst = True if config.authenticator: - logger.warning("Specifying an authenticator doesn't make sense in install mode") - + logger.warning("Specifying an authenticator doesn't make sense when " + "running Certbot with verb \"%s\"", verb) # Try to meet the user's request and/or ask them to pick plugins authenticator = installer = None if verb == "run" and req_auth == req_inst: @@ -189,7 +195,7 @@ def choose_configurator_plugins(config, plugins, verb): authenticator = installer = pick_configurator(config, req_inst, plugins) else: if need_inst or req_inst: - installer = pick_installer(config, req_inst, plugins) + installer = pick_installer(config, req_inst, plugins, installer_question) if need_auth: authenticator = pick_authenticator(config, req_auth, plugins) logger.debug("Selected authenticator %s and installer %s", authenticator, installer) diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 675b936b9..6ec1d4f5c 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -569,5 +569,103 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): self.assertRaises(errors.OverlappingMatchFound, self._call, self.config, None, None, None) +class GetCertnameTest(unittest.TestCase): + """Tests for certbot.cert_manager.""" + + def setUp(self): + self.get_utility_patch = test_util.patch_get_utility() + self.mock_get_utility = self.get_utility_patch.start() + self.config = mock.MagicMock() + self.config.certname = None + + def tearDown(self): + self.get_utility_patch.stop() + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + prompt = "Which certificate would you" + self.mock_get_utility().menu.return_value = (display_util.OK, 0) + self.assertEquals( + cert_manager.get_certnames( + self.config, "verb", allow_multiple=False), ['example.com']) + self.assertTrue( + prompt in self.mock_get_utility().menu.call_args[0][0]) + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames_custom_prompt(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + prompt = "custom prompt" + self.mock_get_utility().menu.return_value = (display_util.OK, 0) + self.assertEquals( + cert_manager.get_certnames( + self.config, "verb", allow_multiple=False, custom_prompt=prompt), + ['example.com']) + self.assertEquals(self.mock_get_utility().menu.call_args[0][0], + prompt) + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames_user_abort(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + self.mock_get_utility().menu.return_value = (display_util.CANCEL, 0) + self.assertRaises( + errors.Error, + cert_manager.get_certnames, + self.config, "erroring_anyway", allow_multiple=False) + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames_allow_multiple(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + prompt = "Which certificate(s) would you" + self.mock_get_utility().checklist.return_value = (display_util.OK, + ['example.com']) + self.assertEquals( + cert_manager.get_certnames( + self.config, "verb", allow_multiple=True), ['example.com']) + self.assertTrue( + prompt in self.mock_get_utility().checklist.call_args[0][0]) + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames_allow_multiple_custom_prompt(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + prompt = "custom prompt" + self.mock_get_utility().checklist.return_value = (display_util.OK, + ['example.com']) + self.assertEquals( + cert_manager.get_certnames( + self.config, "verb", allow_multiple=True, custom_prompt=prompt), + ['example.com']) + self.assertEquals( + self.mock_get_utility().checklist.call_args[0][0], + prompt) + + @mock.patch('certbot.storage.renewal_conf_files') + @mock.patch('certbot.storage.lineagename_for_filename') + def test_get_certnames_allow_multiple_user_abort(self, mock_name, mock_files): + mock_files.return_value = ['example.com.conf'] + mock_name.return_value = 'example.com' + from certbot import cert_manager + self.mock_get_utility().checklist.return_value = (display_util.CANCEL, []) + self.assertRaises( + errors.Error, + cert_manager.get_certnames, + self.config, "erroring_anyway", allow_multiple=True) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 0f2c58161..6add141d4 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -433,6 +433,22 @@ class EnhanceConfigTest(ClientTestCommon): self.client.installer.enhance.assert_not_called() mock_enhancements.ask.assert_not_called() + @mock.patch("certbot.client.logger") + def test_already_exists_header(self, mock_log): + self.config.hsts = True + self._test_with_already_existing() + self.assertTrue(mock_log.warning.called) + self.assertEquals(mock_log.warning.call_args[0][1], + 'Strict-Transport-Security') + + @mock.patch("certbot.client.logger") + def test_already_exists_redirect(self, mock_log): + self.config.redirect = True + self._test_with_already_existing() + self.assertTrue(mock_log.warning.called) + self.assertEquals(mock_log.warning.call_args[0][1], + 'redirect') + def test_no_ask_hsts(self): self.config.hsts = True self._test_with_all_supported() @@ -508,6 +524,13 @@ class EnhanceConfigTest(ClientTestCommon): self.assertEqual(self.client.installer.save.call_count, 1) self.assertEqual(self.client.installer.restart.call_count, 1) + def _test_with_already_existing(self): + self.client.installer = mock.MagicMock() + self.client.installer.supported_enhancements.return_value = [ + "ensure-http-header", "redirect", "staple-ocsp"] + self.client.installer.enhance.side_effect = errors.PluginEnhancementAlreadyPresent() + self.client.enhance_config([self.domain], None) + class RollbackTest(unittest.TestCase): """Tests for certbot.client.rollback.""" diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index c4f58ba7c..9de8c5e9a 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -207,9 +207,9 @@ class ChooseNamesTest(unittest.TestCase): self.mock_install = mock.MagicMock() @classmethod - def _call(cls, installer): + def _call(cls, installer, question=None): from certbot.display.ops import choose_names - return choose_names(installer) + return choose_names(installer, question) @mock.patch("certbot.display.ops._choose_names_manually") def test_no_installer(self, mock_manual): @@ -281,6 +281,15 @@ class ChooseNamesTest(unittest.TestCase): self.assertEqual(names, ["example.com"]) self.assertEqual(mock_util().checklist.call_count, 1) + @test_util.patch_get_utility("certbot.display.ops.z_util") + def test_filter_namees_override_question(self, mock_util): + self.mock_install.get_all_names.return_value = set(["example.com"]) + mock_util().checklist.return_value = (display_util.OK, ["example.com"]) + names = self._call(self.mock_install, "Custom") + self.assertEqual(names, ["example.com"]) + self.assertEqual(mock_util().checklist.call_count, 1) + self.assertEqual(mock_util().checklist.call_args[0][0], "Custom") + @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_nothing_selected(self, mock_util): self.mock_install.get_all_names.return_value = set(["example.com"]) @@ -481,5 +490,42 @@ class ValidatorTests(unittest.TestCase): self.__validator, "msg", default="") +class ChooseValuesTest(unittest.TestCase): + """Test choose_values.""" + @classmethod + def _call(cls, values, question): + from certbot.display.ops import choose_values + return choose_values(values, question) + + @test_util.patch_get_utility("certbot.display.ops.z_util") + def test_choose_names_success(self, mock_util): + items = ["first", "second", "third"] + mock_util().checklist.return_value = (display_util.OK, [items[2]]) + result = self._call(items, None) + self.assertEquals(result, [items[2]]) + self.assertTrue(mock_util().checklist.called) + self.assertEquals(mock_util().checklist.call_args[0][0], None) + + @test_util.patch_get_utility("certbot.display.ops.z_util") + def test_choose_names_success_question(self, mock_util): + items = ["first", "second", "third"] + question = "Which one?" + mock_util().checklist.return_value = (display_util.OK, [items[1]]) + result = self._call(items, question) + self.assertEquals(result, [items[1]]) + self.assertTrue(mock_util().checklist.called) + self.assertEquals(mock_util().checklist.call_args[0][0], question) + + @test_util.patch_get_utility("certbot.display.ops.z_util") + def test_choose_names_user_cancel(self, mock_util): + items = ["first", "second", "third"] + question = "Want to cancel?" + mock_util().checklist.return_value = (display_util.CANCEL, []) + result = self._call(items, question) + self.assertEquals(result, []) + self.assertTrue(mock_util().checklist.called) + self.assertEquals(mock_util().checklist.call_args[0][0], question) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index b778f05ea..d168552cc 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1533,5 +1533,114 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): strict=self.config.strict_permissions) +class EnhanceTest(unittest.TestCase): + """Tests for certbot.main.enhance.""" + + def setUp(self): + self.get_utility_patch = test_util.patch_get_utility() + self.mock_get_utility = self.get_utility_patch.start() + + def tearDown(self): + self.get_utility_patch.stop() + + def _call(self, args): + plugins = disco.PluginsRegistry.find_all() + config = configuration.NamespaceConfig( + cli.prepare_and_parse_args(plugins, args)) + + with mock.patch('certbot.cert_manager.get_certnames') as mock_certs: + mock_certs.return_value = ['example.com'] + with mock.patch('certbot.cert_manager.domains_for_certname') as mock_dom: + mock_dom.return_value = ['example.com'] + with mock.patch('certbot.main._init_le_client') as mock_init: + mock_client = mock.MagicMock() + mock_client.config = config + mock_init.return_value = mock_client + main.enhance(config, plugins) + return mock_client # returns the client + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main._find_domains_or_certname') + def test_selection_question(self, mock_find, mock_choose, mock_lineage, _rec): + mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") + mock_choose.return_value = ['example.com'] + mock_find.return_value = (None, None) + with mock.patch('certbot.main.plug_sel.pick_installer') as mock_pick: + self._call(['enhance', '--redirect']) + self.assertTrue(mock_pick.called) + # Check that the message includes "enhancements" + self.assertTrue("enhancements" in mock_pick.call_args[0][3]) + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main._find_domains_or_certname') + def test_selection_auth_warning(self, mock_find, mock_choose, mock_lineage, _rec): + mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") + mock_choose.return_value = ["example.com"] + mock_find.return_value = (None, None) + with mock.patch('certbot.main.plug_sel.pick_installer'): + with mock.patch('certbot.main.plug_sel.logger.warning') as mock_log: + mock_client = self._call(['enhance', '-a', 'webroot', '--redirect']) + self.assertTrue(mock_log.called) + self.assertTrue("make sense" in mock_log.call_args[0][0]) + self.assertTrue(mock_client.enhance_config.called) + + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + def test_enhance_config_call(self, _rec, mock_choose, mock_lineage): + mock_lineage.return_value = mock.MagicMock(chain_path="/tmp/nonexistent") + mock_choose.return_value = ["example.com"] + with mock.patch('certbot.main.plug_sel.pick_installer'): + mock_client = self._call(['enhance', '--redirect', '--hsts']) + req_enh = ["redirect", "hsts"] + not_req_enh = ["uir"] + self.assertTrue(mock_client.enhance_config.called) + self.assertTrue( + all([getattr(mock_client.config, e) for e in req_enh])) + self.assertFalse( + any([getattr(mock_client.config, e) for e in not_req_enh])) + self.assertTrue( + "example.com" in mock_client.enhance_config.call_args[0][0]) + + @mock.patch('certbot.cert_manager.lineage_for_certname') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + def test_enhance_noninteractive(self, _rec, mock_choose, mock_lineage): + mock_lineage.return_value = mock.MagicMock( + chain_path="/tmp/nonexistent") + mock_choose.return_value = ["example.com"] + with mock.patch('certbot.main.plug_sel.pick_installer'): + mock_client = self._call(['enhance', '--redirect', + '--hsts', '--non-interactive']) + self.assertTrue(mock_client.enhance_config.called) + self.assertFalse(mock_choose.called) + + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + def test_user_abort_domains(self, _rec, mock_choose): + mock_choose.return_value = [] + with mock.patch('certbot.main.plug_sel.pick_installer'): + self.assertRaises(errors.Error, + self._call, + ['enhance', '--redirect', '--hsts']) + + def test_no_enhancements_defined(self): + self.assertRaises(errors.MisconfigurationError, + self._call, ['enhance']) + + @mock.patch('certbot.main.plug_sel.choose_configurator_plugins') + @mock.patch('certbot.main.display_ops.choose_values') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + def test_plugin_selection_error(self, _rec, mock_choose, mock_pick): + mock_choose.return_value = ["example.com"] + mock_pick.return_value = (None, None) + mock_pick.side_effect = errors.PluginSelectionError() + mock_client = self._call(['enhance', '--hsts']) + self.assertFalse(mock_client.enhance_config.called) + if __name__ == '__main__': unittest.main() # pragma: no cover From 398bd4a2cd8e7a3b5f47b6a24fc3a0c604d0be5b Mon Sep 17 00:00:00 2001 From: Jeremy Gillula Date: Wed, 18 Apr 2018 16:46:39 -0700 Subject: [PATCH 14/93] =?UTF-8?q?Emphasizing=20the=20warnings=20in=20the?= =?UTF-8?q?=20READMEs=20in=20/etc/letsencrypt/live/exam=E2=80=A6=20(#5871)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Emphasizing the warnings in the READMEs in /etc/letsencrypt/live/example.com * Making the warning more of a statement --- certbot/storage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/certbot/storage.py b/certbot/storage.py index ed3922c58..c453e55b0 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -1053,6 +1053,9 @@ class RenewableCert(object): "`cert.pem` : will break many server configurations, and " "should not be used\n" " without reading further documentation (see link below).\n\n" + "WARNING: DO NOT MOVE THESE FILES!\n" + " Certbot expects these files to remain in this location in order\n" + " to function properly!\n\n" "We recommend not moving these files. For more information, see the Certbot\n" "User Guide at https://certbot.eff.org/docs/using.html#where-are-my-" "certificates.\n") From f40e04401fba78a54277053f97558a56dd78a882 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 19 Apr 2018 19:35:21 -0400 Subject: [PATCH 15/93] Don't install enum34 when using Python 3.4 or later. Fix #5456. (#5846) The re stdlib module requires attrs that don't exist in the backported 3.4 version. Technically, we are changing our install behavior beyond what is necessary. Previously, enum34 was used for 3.4 and 3.5 as well, and it happened not to conflict, but I think it's better to use the latest bug-fixed stdlib versions as long as they meet the needs of `cryptography`, which is what depends on enum34. That way, at least the various stdlib modules are guaranteed not to conflict with each other. --- letsencrypt-auto-source/letsencrypt-auto | 2 +- letsencrypt-auto-source/pieces/dependency-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 89072ec88..f3525aaee 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1089,7 +1089,7 @@ cryptography==2.0.2 \ --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab -enum34==1.1.2 \ +enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 funcsigs==1.0.2 \ diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 0e2cec984..1e69af9c2 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -93,7 +93,7 @@ cryptography==2.0.2 \ --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab -enum34==1.1.2 \ +enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 funcsigs==1.0.2 \ From 726f3ce8b3540f6f0455a4fcbec1062d17e8cf25 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 19 Apr 2018 17:57:41 -0700 Subject: [PATCH 16/93] Remove EOL'd Ubuntu from targets.yaml (#5887) See https://wiki.ubuntu.com/Releases. Ubuntu 15.* repositories have been shut down for months now causing our tests to always fail on these systems. While the tests on Ubuntu 12.04 still work, it has been unsupported by Canonical for almost a year and I don't think we should hamstring ourselves trying to continue to support it ourselves. --- tests/letstest/targets.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 506225f86..9c1aca24e 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -1,16 +1,6 @@ targets: #----------------------------------------------------------------------------- #Ubuntu - - ami: ami-26d5af4c - name: ubuntu15.10 - type: ubuntu - virt: hvm - user: ubuntu - - ami: ami-d92e6bb3 - name: ubuntu15.04LTS - type: ubuntu - virt: hvm - user: ubuntu - ami: ami-7b89cc11 name: ubuntu14.04LTS type: ubuntu @@ -21,11 +11,6 @@ targets: type: ubuntu virt: pv user: ubuntu - - ami: ami-0611546c - name: ubuntu12.04LTS - type: ubuntu - virt: hvm - user: ubuntu #----------------------------------------------------------------------------- # Debian - ami: ami-116d857a From 9c15fd354f0e98e5b1e48b446304fb16cddced68 Mon Sep 17 00:00:00 2001 From: Aleksandr Volochnev Date: Sat, 21 Apr 2018 00:17:05 +0200 Subject: [PATCH 17/93] Updated base image to python:2-alpine3.7 (#5889) Updated base image from python:2-alpine to python:2-alpine3.7. Python:2-alpine internally utilises alpine version 3.4 which end-of-life date is the first of May 2018. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cf4e9afad..28cd6b323 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2-alpine +FROM python:2-alpine3.7 ENTRYPOINT [ "certbot" ] EXPOSE 80 443 From f510f4bddf4da1411c1966a9cbed2b7e630cba30 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 24 Apr 2018 06:38:15 -0700 Subject: [PATCH 18/93] Update good volunteer task to good first issue. (#5891) --- docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 2098f7cdf..cbb7650b6 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -63,7 +63,7 @@ Find issues to work on ---------------------- You can find the open issues in the `github issue tracker`_. Comparatively -easy ones are marked `Good Volunteer Task`_. If you're starting work on +easy ones are marked `good first issue`_. If you're starting work on something, post a comment to let others know and seek feedback on your plan where appropriate. @@ -72,7 +72,7 @@ your pull request must have thorough unit test coverage, pass our tests, and be compliant with the :ref:`coding style `. .. _github issue tracker: https://github.com/certbot/certbot/issues -.. _Good Volunteer Task: https://github.com/certbot/certbot/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+Volunteer+Task%22 +.. _good first issue: https://github.com/certbot/certbot/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22 .. _testing: From bf30226c697402adac656bfd7956395fc104714b Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 25 Apr 2018 15:09:50 -0700 Subject: [PATCH 19/93] Restore parallel waiting to Route53 plugin (#5712) * Bring back parallel updates to route53. * Re-add try * Fix TTL. * Remove unnecessary wait. * Add pylint exceptions. * Add dummy perform. * review.feedback * Fix underscore. * Fix lint. --- .../certbot_dns_route53/dns_route53.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index 08b1d03f0..27d185656 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -42,14 +42,26 @@ class Authenticator(dns_common.DNSAuthenticator): def _setup_credentials(self): pass - def _perform(self, domain, validation_domain_name, validation): - try: - change_id = self._change_txt_record("UPSERT", validation_domain_name, validation) + def _perform(self, domain, validation_domain_name, validation): # pylint: disable=missing-docstring + pass - self._wait_for_change(change_id) + def perform(self, achalls): + self._attempt_cleanup = True + + try: + change_ids = [ + self._change_txt_record("UPSERT", + achall.validation_domain_name(achall.domain), + achall.validation(achall.account_key)) + for achall in achalls + ] + + for change_id in change_ids: + self._wait_for_change(change_id) except (NoCredentialsError, ClientError) as e: logger.debug('Encountered error during perform: %s', e, exc_info=True) raise errors.PluginError("\n".join([str(e), INSTRUCTIONS])) + return [achall.response(achall.account_key) for achall in achalls] def _cleanup(self, domain, validation_domain_name, validation): try: From 4b870ef940037f7978af31f014c65d5f0e897366 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 1 May 2018 16:59:32 -0700 Subject: [PATCH 20/93] Release 0.24.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 28 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 7 ++++- letsencrypt-auto | 28 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++----- letsencrypt-auto-source/letsencrypt-auto | 26 ++++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 +++++++-------- 22 files changed, 82 insertions(+), 77 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 9cc616c36..98985b18e 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 1ad3b7e43..88f112551 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 061b0c8f3..0848080b3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.23.0" +LE_AUTO_VERSION="0.24.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1089,7 +1089,7 @@ cryptography==2.0.2 \ --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab -enum34==1.1.2 \ +enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 funcsigs==1.0.2 \ @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.23.0 \ - --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ - --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e -acme==0.23.0 \ - --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ - --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 -certbot-apache==0.23.0 \ - --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ - --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e -certbot-nginx==0.23.0 \ - --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ - --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 +certbot==0.24.0 \ + --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ + --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 +acme==0.24.0 \ + --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ + --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb +certbot-apache==0.24.0 \ + --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ + --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 +certbot-nginx==0.24.0 \ + --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ + --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 93495d22e..a5bbf880d 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 7f7eab517..28740ccef 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 6a411dedf..9d35251ee 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 0195fb0da..5be076ba4 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index e004ef92c..29d981874 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 75f588825..2e6f430c5 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index d8a8025ef..8ba9ac7db 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index cebe69b42..5d9ab5fbf 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index a86d12819..7a96bb5fd 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 82f813197..766bc7665 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index b584b872f..9eafd6d4e 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 77ea4170a..6d88e220c 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0.dev0' +version = '0.24.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 9dbab3b70..430535561 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.24.0.dev0' +__version__ = '0.24.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 218e21a88..259142e62 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -9,6 +9,7 @@ obtain, install, and renew certificates: (default) run Obtain & install a certificate in your current webserver certonly Obtain or renew a certificate, but do not install it renew Renew all previously obtained certificates that are near expiry + enhance Add security enhancements to your existing configuration -d DOMAINS Comma-separated list of domains to obtain a certificate for --apache Use the Apache plugin for authentication & installation @@ -107,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.23.0 (certbot; + "". (default: CertbotACMEClient/0.24.0 (certbot; darwin 10.13.4) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags encoded in the user agent are: --duplicate, --force- @@ -397,6 +398,10 @@ update_symlinks: Recreates certificate and key symlinks in /etc/letsencrypt/live, if you changed them by hand or edited a renewal configuration file +enhance: + Helps to harden the TLS configration by adding security enhancements to + already existing configuration. + plugins: Plugin Selection: Certbot client supports an extensible plugins architecture. See 'certbot plugins' for a list of all installed plugins diff --git a/letsencrypt-auto b/letsencrypt-auto index 061b0c8f3..0848080b3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.23.0" +LE_AUTO_VERSION="0.24.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1089,7 +1089,7 @@ cryptography==2.0.2 \ --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \ --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \ --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab -enum34==1.1.2 \ +enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 funcsigs==1.0.2 \ @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.23.0 \ - --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ - --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e -acme==0.23.0 \ - --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ - --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 -certbot-apache==0.23.0 \ - --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ - --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e -certbot-nginx==0.23.0 \ - --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ - --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 +certbot==0.24.0 \ + --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ + --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 +acme==0.24.0 \ + --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ + --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb +certbot-apache==0.24.0 \ + --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ + --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 +certbot-nginx==0.24.0 \ + --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ + --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 620739670..641ebaef8 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlrFS/EACgkQTRfJlc2X -dfK+rQf8DcKY5bMi5eJnwwAlui6WIyWSrf1KAKt09tEGZSHQ1fcyCPrGVhk7VVDg -NJ1/XiYBquPW+7mYUcHrIRsiKYbTUcmVjyqP6tZd67IxRH9ToNqBzA6kq99T+IPd -iTGdczHMSPcxM6/Fa5PYMHXy2+ctTr/8+gnsxth9QfcM62Yd6ecfqIdoId3vk9Aw -UBMENZhUasIvgZDWuow+1XVZ/DAmdvj2Xl/E3sA9i2ArREJhkhVegtdrHkwSY+Hm -MKfZGqNVse6ZAF/8YdEVBum0OngMMs63DwucwFxmw5DqWtmnXm6awLNW/LQ/3R5L -xuKjcVaAT1h5TgIyRT6opH8JBKmLpg== -=Ouj4 +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlro/1AACgkQTRfJlc2X +dfLm5ggAxCrWU9dmYZKllcFzp7TFOdRap0pmarfL4gwSYj7B/bSceD7ysOyoQ8Ra +7UHuZKAQyurZn1seN49d88Kgor9KWZQ1jZiGkfiEpp8qAkdWzFR8UqYa2/CZtk2l +bExm8YQDwhuKvCObGLDGi3ydcIQpfg/rsBkSTphKYXN/Zebx9mAelZN4CgGRy03Y +3z2UqqnyqFPAg4wUGcNfCgUEbJ5bUPr733vQzjBS2IVUbDbu06/1Y8oYzurezXNS +6lEyvTfC5G8RGlSWupNu7yWviD14M4LnAo6WXWEVH+C+ssJaPrZVhZ6KfEt/Erg3 +k06WZSPDCtOm5EfhDm0Rumqm1owA2g== +=Bc4G -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index f3525aaee..0848080b3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.24.0.dev0" +LE_AUTO_VERSION="0.24.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1199,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.23.0 \ - --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ - --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e -acme==0.23.0 \ - --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ - --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 -certbot-apache==0.23.0 \ - --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ - --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e -certbot-nginx==0.23.0 \ - --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ - --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 +certbot==0.24.0 \ + --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ + --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 +acme==0.24.0 \ + --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ + --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb +certbot-apache==0.24.0 \ + --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ + --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 +certbot-nginx==0.24.0 \ + --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ + --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 738fa36e074721d2d7ea6d34b8dafc5937bbf2d1..4a937e7e0c442056410a4d75bc6fc64fa7fc5ed8 100644 GIT binary patch literal 256 zcmV+b0ssDE4g*`fFs}k@&Ip7vL<-AyWT*hCDr+-?vC;X|s(C>trU<`!&JJ7Yet?gj zP<-mLp*eOt8&aten_HT4Xaq=Kx#zQxgGH^wqLqmDbiNH^#Z>!uQ#lxbz9TmzQ``gA zepS_2=ZD+oI|Kl|@HhjznV9>uJqAGPzsGZ`_Tr$cOh5JZM+hOCqkF1gQK5rq?01Ha z$#L4=i}~HfWo>cf%K<$CERWB0i$y+n$+{zxu0TOGArrjm@c)#iwau&wz!kfWn^wMB GGm3kHFnBTS{idqO_gjrF5G{ zVRP?&R3bm%6xg|eMaq^mxiODsYes`VLVP%?>YI`_vpQkyqmump!{L*LC1Srlny=7{ zo3Y>fVHPDtRA0e84`j9mZXT*pgZ6cioA*IXpnwBfgM|DH+$uv9hdg9*BU!VA0HaH%xvk86LU? z;y5! diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 0c1161db7..fc4457771 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.23.0 \ - --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \ - --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e -acme==0.23.0 \ - --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \ - --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550 -certbot-apache==0.23.0 \ - --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \ - --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e -certbot-nginx==0.23.0 \ - --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \ - --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324 +certbot==0.24.0 \ + --hash=sha256:a3fc41fde4f0dbb35f7ebec2f9e00339580b3f4298850411eac0719223073b27 \ + --hash=sha256:a072d4528bb3ac4184f5c961a96931795ddfe4b7cb0f3a98954bdd4cce5f6d70 +acme==0.24.0 \ + --hash=sha256:b92b16102051f447abb52917638fbfb34b646ac07267fee85961b360a0149e32 \ + --hash=sha256:d655e0627c0830114ab3f6732d8bf2f4a2c36f602e0cde10988684e229b501cb +certbot-apache==0.24.0 \ + --hash=sha256:fe54db3e7e09ffe1139041c23ff5123e80aa1526d6fcd40b2a593d005cfcf152 \ + --hash=sha256:686c6c0af5ae8d06e37cc762de7ffa0dc5c3b1ba06ff7653ef61713fa016f891 +certbot-nginx==0.24.0 \ + --hash=sha256:d44c419f72c2cc30de3b138a2cf92e0531696dcb048f287036e229dce2131c00 \ + --hash=sha256:3283d1db057261f05537fa408baee20e0ab9e81c3d55cfba70afe3805cd6f941 From 0ec0d79c3560db13ed86a2ef8ae780655d14f37d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 1 May 2018 16:59:48 -0700 Subject: [PATCH 21/93] Bump version to 0.25.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 98985b18e..72ab5919b 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 88f112551..f0c20da7c 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index a5bbf880d..50df2a56e 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 28740ccef..8e1f9d28b 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 9d35251ee..05998ee6a 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 5be076ba4..cd3b0613e 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 29d981874..10ee710cd 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2e6f430c5..a7f44b989 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8ba9ac7db..c171e5014 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 5d9ab5fbf..2c0e35308 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 7a96bb5fd..821a40655 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 766bc7665..21d9dec29 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 9eafd6d4e..84b701638 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6d88e220c..6889d06e4 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.24.0' +version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index 430535561..27c63e266 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.24.0' +__version__ = '0.25.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0848080b3..853c66023 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.24.0" +LE_AUTO_VERSION="0.25.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 552bfa5eb70bb339f4274b412fac76037fe629dd Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 2 May 2018 10:52:54 +0300 Subject: [PATCH 22/93] New interfaces for installers to run tasks on renew verb (#5879) * ServerTLSUpdater and InstallerSpecificUpdater implementation * Fixed tests and added disables for linter :/ * Added error logging for misconfigurationerror from plugin check * Remove redundant parameter from interfaces * Renaming the interfaces * Finalize interface renaming and move tests to own file * Refactored the runners * Refactor the cli params * Fix the interface args * Fixed documentation * Documentation and naming fixes * Remove ServerTLSConfigurationUpdater * Remove unnecessary linter disable * Rename run_renewal_updaters to run_generic_updaters * Do not raise exception, but make log message more informative and visible for the user * Run renewal deployer before installer restart --- certbot/cli.py | 8 ++++ certbot/constants.py | 1 + certbot/interfaces.py | 75 +++++++++++++++++++++++++++++ certbot/main.py | 6 ++- certbot/renewal.py | 11 +++-- certbot/tests/main_test.py | 17 ++++++- certbot/tests/renewupdater_test.py | 76 ++++++++++++++++++++++++++++++ certbot/updater.py | 67 ++++++++++++++++++++++++++ 8 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 certbot/tests/renewupdater_test.py create mode 100644 certbot/updater.py diff --git a/certbot/cli.py b/certbot/cli.py index 9584c3904..b71d60055 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1192,6 +1192,14 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis default=flag_default("directory_hooks"), dest="directory_hooks", help="Disable running executables found in Certbot's hook directories" " during renewal. (default: False)") + helpful.add( + "renew", "--disable-renew-updates", action="store_true", + default=flag_default("disable_renew_updates"), dest="disable_renew_updates", + help="Disable automatic updates to your server configuration that" + " would otherwise be done by the selected installer plugin, and triggered" + " when the user executes \"certbot renew\", regardless of if the certificate" + " is renewed. This setting does not apply to important TLS configuration" + " updates.") helpful.add_deprecated_argument("--agree-dev-preview", 0) helpful.add_deprecated_argument("--dialog", 0) diff --git a/certbot/constants.py b/certbot/constants.py index 0d0ee8d3f..40557d287 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -64,6 +64,7 @@ CLI_DEFAULTS = dict( pref_challs=[], validate_hooks=True, directory_hooks=True, + disable_renew_updates=False, # Subparsers num=None, diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 501a5c57e..8061f0de3 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -256,6 +256,10 @@ class IConfig(zope.interface.Interface): "user; only needed if your config is somewhere unsafe like /tmp/." "This is a boolean") + disable_renew_updates = zope.interface.Attribute( + "If updates provided by installer enhancements when Certbot is being run" + " with \"renew\" verb should be disabled.") + class IInstaller(IPlugin): """Generic Certbot Installer Interface. @@ -591,3 +595,74 @@ class IReporter(zope.interface.Interface): def print_messages(self): """Prints messages to the user and clears the message queue.""" + + +# Updater interfaces +# +# When "certbot renew" is run, Certbot will iterate over each lineage and check +# if the selected installer for that lineage is a subclass of each updater +# class. If it is and the update of that type is configured to be run for that +# lineage, the relevant update function will be called for each domain in the +# lineage. These functions are never called for other subcommands, so if an +# installer wants to perform an update during the run or install subcommand, it +# should do so when :func:`IInstaller.deploy_cert` is called. + +class GenericUpdater(object): + """Interface for update types not currently specified by Certbot. + + This class allows plugins to perform types of updates that Certbot hasn't + defined (yet). + + To make use of this interface, the installer should implement the interface + methods, and interfaces.GenericUpdater.register(InstallerClass) should + be called from the installer code. + + """ + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def generic_updates(self, domain, *args, **kwargs): + """Perform any update types defined by the installer. + + If an installer is a subclass of the class containing this method, this + function will always be called when "certbot renew" is run. If the + update defined by the installer should be run conditionally, the + installer needs to handle checking the conditions itself. + + This method is called once for each domain. + + :param str domain: domain to handle the updates for + + """ + + +class RenewDeployer(object): + """Interface for update types run when a lineage is renewed + + This class allows plugins to perform types of updates that need to run at + lineage renewal that Certbot hasn't defined (yet). + + To make use of this interface, the installer should implement the interface + methods, and interfaces.RenewDeployer.register(InstallerClass) should + be called from the installer code. + """ + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def renew_deploy(self, lineage, *args, **kwargs): + """Perform updates defined by installer when a certificate has been renewed + + If an installer is a subclass of the class containing this method, this + function will always be called when a certficate has been renewed by + running "certbot renew". For example if a plugin needs to copy a + certificate over, or change configuration based on the new certificate. + + This method is called once for each lineage renewed + + :param lineage: Certificate lineage object that is set if certificate + was renewed on this run. + :type lineage: storage.RenewableCert + + """ diff --git a/certbot/main.py b/certbot/main.py index dd0991c8d..a041b998f 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -29,6 +29,7 @@ from certbot import log from certbot import renewal from certbot import reporter from certbot import storage +from certbot import updater from certbot import util from certbot.display import util as display_util, ops as display_ops @@ -1145,10 +1146,9 @@ def renew_cert(config, plugins, lineage): except errors.PluginSelectionError as e: logger.info("Could not choose appropriate plugin: %s", e) raise - le_client = _init_le_client(config, auth, installer) - _get_and_save_cert(le_client, config, lineage=lineage) + renewed_lineage = _get_and_save_cert(le_client, config, lineage=lineage) notify = zope.component.getUtility(interfaces.IDisplay).notification if installer is None: @@ -1158,9 +1158,11 @@ def renew_cert(config, plugins, lineage): # In case of a renewal, reload server to pick up new certificate. # In principle we could have a configuration option to inhibit this # from happening. + updater.run_renewal_deployer(renewed_lineage, installer, config) installer.restart() notify("new certificate deployed with reload of {0} server; fullchain is {1}".format( config.installer, lineage.fullchain), pause=False) + # Run deployer def certonly(config, plugins): """Authenticate & obtain cert, but do not install it. diff --git a/certbot/renewal.py b/certbot/renewal.py index ea5d87a5e..4651eeb36 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -12,13 +12,14 @@ import zope.component import OpenSSL from certbot import cli - from certbot import crypto_util from certbot import errors from certbot import interfaces from certbot import util from certbot import hooks from certbot import storage +from certbot import updater + from certbot.plugins import disco as plugins_disco logger = logging.getLogger(__name__) @@ -411,9 +412,9 @@ def handle_renewal_request(config): # XXX: ensure that each call here replaces the previous one zope.component.provideUtility(lineage_config) renewal_candidate.ensure_deployed() + from certbot import main + plugins = plugins_disco.PluginsRegistry.find_all() if should_renew(lineage_config, renewal_candidate): - plugins = plugins_disco.PluginsRegistry.find_all() - from certbot import main # domains have been restored into lineage_config by reconstitute # but they're unnecessary anyway because renew_cert here # will just grab them from the certificate @@ -426,6 +427,10 @@ def handle_renewal_request(config): "cert", renewal_candidate.latest_common_version())) renew_skipped.append("%s expires on %s" % (renewal_candidate.fullchain, expiry.strftime("%Y-%m-%d"))) + # Run updater interface methods + updater.run_generic_updaters(lineage_config, plugins, + renewal_candidate) + except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. logger.warning("Attempting to renew cert (%s) from %s produced an " diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index d168552cc..22653ca3a 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -23,6 +23,7 @@ from certbot import configuration from certbot import crypto_util from certbot import errors from certbot import main +from certbot import updater from certbot import util from certbot.plugins import disco @@ -1232,7 +1233,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._test_renew_common(renewalparams=renewalparams, error_expected=True, names=names, assert_oc_called=False) - def test_renew_with_configurator(self): + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + def test_renew_with_configurator(self, mock_sel): + mock_sel.return_value = (mock.MagicMock(), mock.MagicMock()) renewalparams = {'authenticator': 'webroot'} self._test_renew_common( renewalparams=renewalparams, assert_oc_called=True, @@ -1448,6 +1451,18 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met email in mock_utility().add_message.call_args[0][0]) self.assertTrue(mock_handle.called) + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + def test_plugin_selection_error(self, mock_choose): + mock_choose.side_effect = errors.PluginSelectionError + self.assertRaises(errors.PluginSelectionError, main.renew_cert, + None, None, None) + + with mock.patch('certbot.updater.logger.warning') as mock_log: + updater.run_generic_updaters(None, None, None) + self.assertTrue(mock_log.called) + self.assertTrue("Could not choose appropriate plugin for updaters" + in mock_log.call_args[0][0]) + class UnregisterTest(unittest.TestCase): def setUp(self): diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py new file mode 100644 index 000000000..9d0f8d515 --- /dev/null +++ b/certbot/tests/renewupdater_test.py @@ -0,0 +1,76 @@ +"""Tests for renewal updater interfaces""" +import unittest +import mock + +from certbot import interfaces +from certbot import main +from certbot import updater + +import certbot.tests.util as test_util + + +class RenewUpdaterTest(unittest.TestCase): + """Tests for interfaces.RenewDeployer and interfaces.GenericUpdater""" + + def setUp(self): + class MockInstallerGenericUpdater(interfaces.GenericUpdater): + """Mock class that implements GenericUpdater""" + def __init__(self, *args, **kwargs): + # pylint: disable=unused-argument + self.restart = mock.MagicMock() + self.callcounter = mock.MagicMock() + def generic_updates(self, domain, *args, **kwargs): + self.callcounter(*args, **kwargs) + + class MockInstallerRenewDeployer(interfaces.RenewDeployer): + """Mock class that implements RenewDeployer""" + def __init__(self, *args, **kwargs): + # pylint: disable=unused-argument + self.callcounter = mock.MagicMock() + def renew_deploy(self, lineage, *args, **kwargs): + self.callcounter(*args, **kwargs) + + self.generic_updater = MockInstallerGenericUpdater() + self.renew_deployer = MockInstallerRenewDeployer() + + def get_config(self, args): + """Get mock config from dict of parameters""" + config = mock.MagicMock() + for key in args.keys(): + config.__dict__[key] = args[key] + return config + + @mock.patch('certbot.main._get_and_save_cert') + @mock.patch('certbot.plugins.selection.choose_configurator_plugins') + @test_util.patch_get_utility() + def test_server_updates(self, _, mock_select, mock_getsave): + config = self.get_config({"disable_renew_updates": False}) + + lineage = mock.MagicMock() + lineage.names.return_value = ['firstdomain', 'seconddomain'] + mock_getsave.return_value = lineage + mock_generic_updater = self.generic_updater + + # Generic Updater + mock_select.return_value = (mock_generic_updater, None) + with mock.patch('certbot.main._init_le_client'): + main.renew_cert(config, None, mock.MagicMock()) + self.assertTrue(mock_generic_updater.restart.called) + + mock_generic_updater.restart.reset_mock() + mock_generic_updater.callcounter.reset_mock() + updater.run_generic_updaters(config, None, lineage) + self.assertEqual(mock_generic_updater.callcounter.call_count, 2) + self.assertFalse(mock_generic_updater.restart.called) + + def test_renew_deployer(self): + config = self.get_config({"disable_renew_updates": False}) + lineage = mock.MagicMock() + lineage.names.return_value = ['firstdomain', 'seconddomain'] + mock_deployer = self.renew_deployer + updater.run_renewal_deployer(lineage, mock_deployer, config) + self.assertTrue(mock_deployer.callcounter.called_with(lineage)) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/certbot/updater.py b/certbot/updater.py new file mode 100644 index 000000000..f822c55ee --- /dev/null +++ b/certbot/updater.py @@ -0,0 +1,67 @@ +"""Updaters run at renewal""" +import logging + +from certbot import errors +from certbot import interfaces + +from certbot.plugins import selection as plug_sel + +logger = logging.getLogger(__name__) + +def run_generic_updaters(config, plugins, lineage): + """Run updaters that the plugin supports + + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: List of plugins + :type plugins: `list` of `str` + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :returns: `None` + :rtype: None + """ + try: + # installers are used in auth mode to determine domain names + installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "certonly") + except errors.PluginSelectionError as e: + logger.warning("Could not choose appropriate plugin for updaters: %s", e) + return + _run_updaters(lineage, installer, config) + +def run_renewal_deployer(lineage, installer, config): + """Helper function to run deployer interface method if supported by the used + installer plugin. + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :returns: `None` + :rtype: None + """ + if not config.disable_renew_updates and isinstance(installer, + interfaces.RenewDeployer): + installer.renew_deploy(lineage) + +def _run_updaters(lineage, installer, config): + """Helper function to run the updater interface methods if supported by the + used installer plugin. + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :param installer: Installer object + :type installer: interfaces.IInstaller + + :returns: `None` + :rtype: None + """ + for domain in lineage.names(): + if not config.disable_renew_updates: + if isinstance(installer, interfaces.GenericUpdater): + installer.generic_updates(domain) From 7fa3455dc6c22b51e7dee4bb33b51d63ca087412 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 2 May 2018 12:18:29 -0700 Subject: [PATCH 23/93] Update changelog for 0.24.0 (#5915) --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89c7c41a9..044e55250 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,49 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.24.0 - 2018-05-02 + +### Added + +* certbot now has an enhance subcommand which allows you to configure security + enhancements like HTTP to HTTPS redirects, OCSP stapling, and HSTS without + reinstalling a certificate. +* certbot-dns-rfc2136 now allows the user to specify the port to use to reach + the DNS server in its credentials file. +* acme now parses the wildcard field included in authorizations so it can be + used by users of the library. + +### Changed + +* certbot-dns-route53 used to wait for each DNS update to propagate before + sending the next one, but now it sends all updates before waiting which + speeds up issuance for multiple domains dramatically. +* Certbot's official Docker images are now based on Alpine Linux 3.7 rather + than 3.4 because 3.4 has reached its end-of-life. +* We've doubled the time Certbot will spend polling authorizations before + timing out. +* The level of the message logged when Certbot is being used with + non-standard paths warning that crontabs for renewal included in Certbot + packages from OS package managers may not work has been reduced. This stops + the message from being written to stderr every time `certbot renew` runs. + +### Fixed + +* certbot-auto now works with Python 3.6. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with changes other than their version number were: + +* acme +* certbot +* certbot-apache +* certbot-dns-digitalocean (only style improvements to tests) +* certbot-dns-rfc2136 + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/52?closed=1 + ## 0.23.0 - 2018-04-04 ### Added From 95c0c4a7083f45e4157485303624eab271024ca0 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 3 May 2018 01:56:37 +0300 Subject: [PATCH 24/93] Py2 and Py3 compatibility for metaclass interfaces (#5913) --- certbot/interfaces.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 8061f0de3..f5858a7fd 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -1,5 +1,6 @@ """Certbot client interfaces.""" import abc +import six import zope.interface # pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class @@ -607,6 +608,7 @@ class IReporter(zope.interface.Interface): # installer wants to perform an update during the run or install subcommand, it # should do so when :func:`IInstaller.deploy_cert` is called. +@six.add_metaclass(abc.ABCMeta) class GenericUpdater(object): """Interface for update types not currently specified by Certbot. @@ -619,8 +621,6 @@ class GenericUpdater(object): """ - __metaclass__ = abc.ABCMeta - @abc.abstractmethod def generic_updates(self, domain, *args, **kwargs): """Perform any update types defined by the installer. @@ -637,6 +637,7 @@ class GenericUpdater(object): """ +@six.add_metaclass(abc.ABCMeta) class RenewDeployer(object): """Interface for update types run when a lineage is renewed @@ -648,8 +649,6 @@ class RenewDeployer(object): be called from the installer code. """ - __metaclass__ = abc.ABCMeta - @abc.abstractmethod def renew_deploy(self, lineage, *args, **kwargs): """Perform updates defined by installer when a certificate has been renewed From 32e85e9a230cd202d6f7cf790164f36c6210396f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 3 May 2018 00:59:25 -0700 Subject: [PATCH 25/93] correct metaclass usage everywhere (#5919) --- acme/acme/challenges.py | 3 ++- certbot/interfaces.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 96997297b..9702a7a14 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -9,6 +9,7 @@ from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose import OpenSSL import requests +import six from acme import errors from acme import crypto_util @@ -139,6 +140,7 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return True +@six.add_metaclass(abc.ABCMeta) class KeyAuthorizationChallenge(_TokenChallenge): # pylint: disable=abstract-class-little-used,too-many-ancestors """Challenge based on Key Authorization. @@ -147,7 +149,6 @@ class KeyAuthorizationChallenge(_TokenChallenge): that will be used to generate `response`. """ - __metaclass__ = abc.ABCMeta response_cls = NotImplemented thumbprint_hash_function = ( diff --git a/certbot/interfaces.py b/certbot/interfaces.py index f5858a7fd..c96f6bd51 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -7,11 +7,10 @@ import zope.interface # pylint: disable=too-few-public-methods +@six.add_metaclass(abc.ABCMeta) class AccountStorage(object): """Accounts storage interface.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod def find_all(self): # pragma: no cover """Find all accounts. From 3eaf35f1e2473187882dc746547886b859870b2b Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 3 May 2018 13:10:33 -0700 Subject: [PATCH 26/93] Check_untyped_defs in mypy with clean output for acme (#5874) * check_untyped_defs in mypy with clean output for acme * test entire acme module * Add typing as a dependency because it's only in the stdlib for 3.5+ * Add str_utils, modified for python2.7 compatibility * make mypy happy in acme * typing is needed in prod * we actually only need typing in acme so far * add tests and more docs for str_utils * pragma no cover * add magic_typing * s/from typing/from magic_typing/g * move typing to dev_extras * correctly set up imports * remove str_utils * only type: ignore for OpenSSL.SSL, not crypto * Since we only run mypy with python3 anyway and we're fine importing it when it's not actually there, there's no actual need for typing to be present as a dependency * comment magic_typing.py * disable wildcard-import im magic_typing * disable pylint errors * add magic_typing_test * make magic_typing tests work alongside other tests * make sure temp_typing is set * add typing as a dev dependency for python3.4 * run mypy with python3.4 on travis to get a little more testing with different environments * don't stick typing into sys.modules * reorder imports --- .travis.yml | 2 ++ acme/acme/challenges.py | 2 +- acme/acme/client.py | 9 ++--- acme/acme/client_test.py | 4 ++- acme/acme/crypto_util.py | 63 ++++++++++++++++++---------------- acme/acme/crypto_util_test.py | 3 +- acme/acme/magic_typing.py | 13 +++++++ acme/acme/magic_typing_test.py | 41 ++++++++++++++++++++++ acme/acme/messages_test.py | 3 +- acme/acme/standalone.py | 9 ++--- acme/acme/standalone_test.py | 5 +-- acme/acme/test_util.py | 14 ++++---- mypy.ini | 3 ++ setup.py | 1 + 14 files changed, 121 insertions(+), 51 deletions(-) create mode 100644 acme/acme/magic_typing.py create mode 100644 acme/acme/magic_typing_test.py diff --git a/.travis.yml b/.travis.yml index 370137f68..111ddb3d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,8 @@ matrix: addons: - python: "2.7" env: TOXENV=lint + - python: "3.4" + env: TOXENV=mypy - python: "3.5" env: TOXENV=mypy - python: "2.7" diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 9702a7a14..b2a4882eb 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -478,7 +478,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): try: cert = self.probe_cert(domain=domain, **kwargs) except errors.Error as error: - logger.debug(error, exc_info=True) + logger.debug(str(error), exc_info=True) return False return self.verify_cert(cert) diff --git a/acme/acme/client.py b/acme/acme/client.py index 19615b087..7807f0ece 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -9,7 +9,6 @@ import time import six from six.moves import http_client # pylint: disable=import-error - import josepy as jose import OpenSSL import re @@ -20,6 +19,8 @@ from acme import crypto_util from acme import errors from acme import jws from acme import messages +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, List, Set, Text logger = logging.getLogger(__name__) @@ -415,7 +416,7 @@ class Client(ClientBase): """ # pylint: disable=too-many-locals assert max_attempts > 0 - attempts = collections.defaultdict(int) + attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int] exhausted = set() # priority queue with datetime.datetime (based on Retry-After) as key, @@ -529,7 +530,7 @@ class Client(ClientBase): :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509` """ - chain = [] + chain = [] # type: List[jose.ComparableX509] uri = certr.cert_chain_uri while uri is not None and len(chain) < max_length: response, cert = self._get_cert(uri) @@ -864,7 +865,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes self.account = account self.alg = alg self.verify_ssl = verify_ssl - self._nonces = set() + self._nonces = set() # type: Set[Text] self.user_agent = user_agent self.session = requests.Session() self._default_timeout = timeout diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index be08c2919..c17b83210 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -17,6 +17,7 @@ from acme import jws as acme_jws from acme import messages from acme import messages_test from acme import test_util +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module CERT_DER = test_util.load_vector('cert.der') @@ -61,7 +62,8 @@ class ClientTestBase(unittest.TestCase): self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212') reg = messages.Registration( contact=self.contact, key=KEY.public_key()) - self.new_reg = messages.NewRegistration(**dict(reg)) + the_arg = dict(reg) # type: Dict + self.new_reg = messages.NewRegistration(**the_arg) # pylint: disable=star-args self.regr = messages.RegistrationResource( body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 2281196eb..ad914ca60 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -6,11 +6,13 @@ import os import re import socket -import OpenSSL +from OpenSSL import crypto +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 import josepy as jose - from acme import errors +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Callable, Text, Union logger = logging.getLogger(__name__) @@ -25,7 +27,7 @@ logger = logging.getLogger(__name__) # https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni # should be changed to use "set_options" to disable SSLv2 and SSLv3, # in case it's used for things other than probing/serving! -_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD # type: ignore +_DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore class SSLSocket(object): # pylint: disable=too-few-public-methods @@ -64,9 +66,9 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods logger.debug("Server name (%s) not recognized, dropping SSL", server_name) return - new_context = OpenSSL.SSL.Context(self.method) - new_context.set_options(OpenSSL.SSL.OP_NO_SSLv2) - new_context.set_options(OpenSSL.SSL.OP_NO_SSLv3) + new_context = SSL.Context(self.method) + new_context.set_options(SSL.OP_NO_SSLv2) + new_context.set_options(SSL.OP_NO_SSLv3) new_context.use_privatekey(key) new_context.use_certificate(cert) connection.set_context(new_context) @@ -89,18 +91,18 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods def accept(self): # pylint: disable=missing-docstring sock, addr = self.sock.accept() - context = OpenSSL.SSL.Context(self.method) - context.set_options(OpenSSL.SSL.OP_NO_SSLv2) - context.set_options(OpenSSL.SSL.OP_NO_SSLv3) + context = SSL.Context(self.method) + context.set_options(SSL.OP_NO_SSLv2) + context.set_options(SSL.OP_NO_SSLv3) context.set_tlsext_servername_callback(self._pick_certificate_cb) - ssl_sock = self.FakeConnection(OpenSSL.SSL.Connection(context, sock)) + ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) ssl_sock.set_accept_state() logger.debug("Performing handshake with %s", addr) try: ssl_sock.do_handshake() - except OpenSSL.SSL.Error as error: + except SSL.Error as error: # _pick_certificate_cb might have returned without # creating SSL context (wrong server name) raise socket.error(error) @@ -128,7 +130,7 @@ def probe_sni(name, host, port=443, timeout=300, :rtype: OpenSSL.crypto.X509 """ - context = OpenSSL.SSL.Context(method) + context = SSL.Context(method) context.set_timeout(timeout) socket_kwargs = {'source_address': source_address} @@ -145,13 +147,13 @@ def probe_sni(name, host, port=443, timeout=300, raise errors.Error(error) with contextlib.closing(sock) as client: - client_ssl = OpenSSL.SSL.Connection(context, client) + client_ssl = SSL.Connection(context, client) client_ssl.set_connect_state() client_ssl.set_tlsext_host_name(name) # pyOpenSSL>=0.13 try: client_ssl.do_handshake() client_ssl.shutdown() - except OpenSSL.SSL.Error as error: + except SSL.Error as error: raise errors.Error(error) return client_ssl.get_peer_certificate() @@ -164,18 +166,18 @@ def make_csr(private_key_pem, domains, must_staple=False): OCSP Must Staple: https://tools.ietf.org/html/rfc7633). :returns: buffer PEM-encoded Certificate Signing Request. """ - private_key = OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, private_key_pem) - csr = OpenSSL.crypto.X509Req() + private_key = crypto.load_privatekey( + crypto.FILETYPE_PEM, private_key_pem) + csr = crypto.X509Req() extensions = [ - OpenSSL.crypto.X509Extension( + crypto.X509Extension( b'subjectAltName', critical=False, value=', '.join('DNS:' + d for d in domains).encode('ascii') ), ] if must_staple: - extensions.append(OpenSSL.crypto.X509Extension( + extensions.append(crypto.X509Extension( b"1.3.6.1.5.5.7.1.24", critical=False, value=b"DER:30:03:02:01:05")) @@ -183,8 +185,8 @@ def make_csr(private_key_pem, domains, must_staple=False): csr.set_pubkey(private_key) csr.set_version(2) csr.sign(private_key, 'sha256') - return OpenSSL.crypto.dump_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, csr) + return crypto.dump_certificate_request( + crypto.FILETYPE_PEM, csr) def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): common_name = loaded_cert_or_req.get_subject().CN @@ -221,11 +223,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req): parts_separator = ", " prefix = "DNS" + part_separator - if isinstance(cert_or_req, OpenSSL.crypto.X509): - func = OpenSSL.crypto.dump_certificate + if isinstance(cert_or_req, crypto.X509): + # pylint: disable=line-too-long + func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] else: - func = OpenSSL.crypto.dump_certificate_request - text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") + func = crypto.dump_certificate_request + text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") # WARNING: this function does not support multiple SANs extensions. # Multiple X509v3 extensions of the same type is disallowed by RFC 5280. match = re.search(r"X509v3 Subject Alternative Name:(?: critical)?\s*(.*)", text) @@ -252,12 +255,12 @@ def gen_ss_cert(key, domains, not_before=None, """ assert domains, "Must provide one or more hostnames for the cert." - cert = OpenSSL.crypto.X509() + cert = crypto.X509() cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16)) cert.set_version(2) extensions = [ - OpenSSL.crypto.X509Extension( + crypto.X509Extension( b"basicConstraints", True, b"CA:TRUE, pathlen:0"), ] @@ -266,7 +269,7 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_issuer(cert.get_subject()) if force_san or len(domains) > 1: - extensions.append(OpenSSL.crypto.X509Extension( + extensions.append(crypto.X509Extension( b"subjectAltName", critical=False, value=b", ".join(b"DNS:" + d.encode() for d in domains) @@ -281,7 +284,7 @@ def gen_ss_cert(key, domains, not_before=None, cert.sign(key, "sha256") return cert -def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): +def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in @@ -298,7 +301,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): if isinstance(cert, jose.ComparableX509): # pylint: disable=protected-access cert = cert.wrapped - return OpenSSL.crypto.dump_certificate(filetype, cert) + return crypto.dump_certificate(filetype, cert) # assumes that OpenSSL.crypto.dump_certificate includes ending # newline character diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 3874ba9d9..36d62b324 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -13,6 +13,7 @@ import OpenSSL from acme import errors from acme import test_util +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module class SSLSocketAndProbeSNITest(unittest.TestCase): @@ -165,7 +166,7 @@ class RandomSnTest(unittest.TestCase): def setUp(self): self.cert_count = 5 - self.serial_num = [] + self.serial_num = [] # type: List[int] self.key = OpenSSL.crypto.PKey() self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py new file mode 100644 index 000000000..555088cf2 --- /dev/null +++ b/acme/acme/magic_typing.py @@ -0,0 +1,13 @@ +"""Shim class to not have to depend on typing module in prod.""" +import sys + +class TypingClass(object): + """Ignore import errors by getting anything""" + def __getattr__(self, name): + return None + +try: + # mypy doesn't respect modifying sys.modules + from typing import * # pylint: disable=wildcard-import, unused-wildcard-import +except ImportError: + sys.modules[__name__] = TypingClass() diff --git a/acme/acme/magic_typing_test.py b/acme/acme/magic_typing_test.py new file mode 100644 index 000000000..23dfe3367 --- /dev/null +++ b/acme/acme/magic_typing_test.py @@ -0,0 +1,41 @@ +"""Tests for acme.magic_typing.""" +import sys +import unittest + +import mock + + +class MagicTypingTest(unittest.TestCase): + """Tests for acme.magic_typing.""" + def test_import_success(self): + try: + import typing as temp_typing + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover + typing_class_mock = mock.MagicMock() + text_mock = mock.MagicMock() + typing_class_mock.Text = text_mock + sys.modules['typing'] = typing_class_mock + if 'acme.magic_typing' in sys.modules: + del sys.modules['acme.magic_typing'] # pragma: no cover + from acme.magic_typing import Text # pylint: disable=no-name-in-module + self.assertEqual(Text, text_mock) + del sys.modules['acme.magic_typing'] + sys.modules['typing'] = temp_typing + + def test_import_failure(self): + try: + import typing as temp_typing + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover + sys.modules['typing'] = None + if 'acme.magic_typing' in sys.modules: + del sys.modules['acme.magic_typing'] # pragma: no cover + from acme.magic_typing import Text # pylint: disable=no-name-in-module + self.assertTrue(Text is None) + del sys.modules['acme.magic_typing'] + sys.modules['typing'] = temp_typing + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 64bc81efd..0e2d8c62d 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -6,6 +6,7 @@ import mock from acme import challenges from acme import test_util +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module CERT = test_util.load_comparable_cert('cert.der') @@ -85,7 +86,7 @@ class ConstantTest(unittest.TestCase): from acme.messages import _Constant class MockConstant(_Constant): # pylint: disable=missing-docstring - POSSIBLE_NAMES = {} + POSSIBLE_NAMES = {} # type: Dict self.MockConstant = MockConstant # pylint: disable=invalid-name self.const_a = MockConstant('a') diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index c221f5883..a370501ee 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -16,6 +16,7 @@ import OpenSSL from acme import challenges from acme import crypto_util +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module logger = logging.getLogger(__name__) @@ -66,8 +67,8 @@ class BaseDualNetworkedServers(object): def __init__(self, ServerClass, server_address, *remaining_args, **kwargs): port = server_address[1] - self.threads = [] - self.servers = [] + self.threads = [] # type: List[threading.Thread] + self.servers = [] # type: List[ACMEServerMixin] # Must try True first. # Ubuntu, for example, will fail to bind to IPv4 if we've already bound @@ -189,7 +190,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.simple_http_resources = kwargs.pop("simple_http_resources", set()) - socketserver.BaseRequestHandler.__init__(self, *args, **kwargs) + BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def log_message(self, format, *args): # pylint: disable=redefined-builtin """Log arbitrary message.""" @@ -262,7 +263,7 @@ def simple_tls_sni_01_server(cli_args, forever=True): certs = {} - _, hosts, _ = next(os.walk('.')) + _, hosts, _ = next(os.walk('.')) # type: ignore # https://github.com/python/mypy/issues/465 for host in hosts: with open(os.path.join(host, "cert.pem")) as cert_file: cert_contents = cert_file.read() diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 5b1b72c18..1591187e5 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -18,6 +18,7 @@ from acme import challenges from acme import crypto_util from acme import errors from acme import test_util +from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module class TLSServerTest(unittest.TestCase): @@ -72,7 +73,7 @@ class HTTP01ServerTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() + self.resources = set() # type: Set from acme.standalone import HTTP01Server self.server = HTTP01Server(('', 0), resources=self.resources) @@ -201,7 +202,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() + self.resources = set() # type: Set from acme.standalone import HTTP01DualNetworkedServers self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources) diff --git a/acme/acme/test_util.py b/acme/acme/test_util.py index d5d57bd27..1a0b67056 100644 --- a/acme/acme/test_util.py +++ b/acme/acme/test_util.py @@ -10,7 +10,7 @@ import unittest from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -import OpenSSL +from OpenSSL import crypto def vector_path(*names): @@ -39,8 +39,8 @@ def _guess_loader(filename, loader_pem, loader_der): def load_cert(*names): """Load certificate.""" loader = _guess_loader( - names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1) - return OpenSSL.crypto.load_certificate(loader, load_vector(*names)) + names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1) + return crypto.load_certificate(loader, load_vector(*names)) def load_comparable_cert(*names): @@ -51,8 +51,8 @@ def load_comparable_cert(*names): def load_csr(*names): """Load certificate request.""" loader = _guess_loader( - names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1) - return OpenSSL.crypto.load_certificate_request(loader, load_vector(*names)) + names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1) + return crypto.load_certificate_request(loader, load_vector(*names)) def load_comparable_csr(*names): @@ -71,8 +71,8 @@ def load_rsa_private_key(*names): def load_pyopenssl_private_key(*names): """Load pyOpenSSL private key.""" loader = _guess_loader( - names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1) - return OpenSSL.crypto.load_privatekey(loader, load_vector(*names)) + names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1) + return crypto.load_privatekey(loader, load_vector(*names)) def skip_unless(condition, reason): # pragma: no cover diff --git a/mypy.ini b/mypy.ini index aac60a2ad..2bac9249f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,3 +1,6 @@ [mypy] python_version = 2.7 ignore_missing_imports = True + +[mypy-acme.*] +check_untyped_defs = True diff --git a/setup.py b/setup.py index e674871a8..3760fd35b 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ dev_extras = [ dev3_extras = [ 'mypy', + 'typing', # for python3.4 ] docs_extras = [ From 83ea820525d4342ee58baa0d6c6547b6843d8dc6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 9 May 2018 12:11:36 -0700 Subject: [PATCH 27/93] disable apacheconftest (#5937) --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 111ddb3d4..fa68d376c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,9 +55,6 @@ matrix: services: docker before_install: addons: - - python: "2.7" - env: TOXENV=apacheconftest - sudo: required - python: "2.7" env: TOXENV=nginxroundtrip From 832941279b23b2da73921c7b89878ebb5fac2fec Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 10 May 2018 16:46:57 -0700 Subject: [PATCH 28/93] Specify that every domain name needs to be under a server_name directive (#5949) --- certbot-nginx/certbot_nginx/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 3ba8bcb06..378f24b63 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -293,7 +293,7 @@ class NginxConfigurator(common.Installer): ("Cannot find a VirtualHost matching domain %s. " "In order for Certbot to correctly perform the challenge " "please add a corresponding server_name directive to your " - "nginx configuration: " + "nginx configuration for every domain on your certificate: " "https://nginx.org/en/docs/http/server_names.html") % (target_name)) # Note: if we are enhancing with ocsp, vhost should already be ssl. for vhost in vhosts: From 8851141dcf93d8e6c080bcd1503e64762fcebf1b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 11 May 2018 06:12:10 -0700 Subject: [PATCH 29/93] Revert "disable apacheconftest (#5937)" (#5954) This reverts commit 83ea820525d4342ee58baa0d6c6547b6843d8dc6. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index fa68d376c..111ddb3d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,6 +55,9 @@ matrix: services: docker before_install: addons: + - python: "2.7" + env: TOXENV=apacheconftest + sudo: required - python: "2.7" env: TOXENV=nginxroundtrip From 2f89a10f50b1a81d3a7fa5fc264fffc521b575dd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 11 May 2018 07:07:02 -0700 Subject: [PATCH 30/93] Small dev docs improvements (#5953) * Clarify UNIX only * Have people develop natively. Some systems like Arch Linux and macOS require --debug in order to install dependencies. Our bootstrapping script for macOS works, so let's let people who want to develop natively. * briefly mention docker as dev option * remove bad _common.sh info * update OS dep section * Remove sudo from certbot-auto usage When sudo isn't available, Certbot is able to fall back to su. Removing it from the instructions here allows the command to work when its run in minimal systems like Docker where sudo may not be installed. * copy advice about missing interpreters * Improve integration tests docs Explain what a boulder is and tell people they probably should just let the tests run in Travis. * Don't tell people to run integration tests. I don't think any paid Certbot devs run integration tests locally and instead rely on Travis. Let's not make others do it. * fix spacing * you wouldn't download a CA * address review comments --- docs/contributing.rst | 104 ++++++++++++++++++------------------------ docs/install.rst | 21 +++++---- 2 files changed, 56 insertions(+), 69 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index cbb7650b6..e86d1a5b3 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -11,6 +11,12 @@ Developer Guide Getting Started =============== +Certbot has the same :ref:`system requirements ` when set +up for development. While the section below will help you install Certbot and +its dependencies, Certbot needs to be run on a UNIX-like OS so if you're using +Windows, you'll need to set up a (virtual) machine running an OS such as Linux +and continue with these instructions on that UNIX-like OS. + Running a local copy of the client ---------------------------------- @@ -22,40 +28,35 @@ running: git clone https://github.com/certbot/certbot -If you're on macOS, we recommend you skip the rest of this section and instead -run Certbot in Docker. You can find instructions for how to do this :ref:`here -`. If you're running on Linux, you can run the following commands to -install dependencies and set up a virtual environment where you can run -Certbot. You will need to repeat this when Certbot's dependencies change or when -a new plugin is introduced. +Next you need to install dependencies and set up a virtual environment where +you can run Certbot. We recommend you do this using the commands below, +however, you can alternatively skip the rest of this section and :ref:`run +Certbot in Docker `. .. code-block:: shell cd certbot - sudo ./certbot-auto --os-packages-only - ./tools/venv.sh + ./certbot-auto --debug --os-packages-only + tools/venv.sh + +.. note:: You may need to repeat this when + Certbot's dependencies change or when a new plugin is introduced. You can now run the copy of Certbot from git either by executing -``venv/bin/certbot``, or by activating the virtual environment. If you're -actively modifying and testing the code, you may want to run commands like this in -each shell where you're working: +``venv/bin/certbot``, or by activating the virtual environment. You can do the +latter by running: .. code-block:: shell - source ./venv/bin/activate - export SERVER=https://acme-staging-v02.api.letsencrypt.org/directory - source tests/integration/_common.sh + source venv/bin/activate -After that, your shell will be using the virtual environment, your copy of -Certbot will default to requesting test (staging) certificates, and you run the -client by typing `certbot` or `certbot_test`. The latter is an alias that -includes several flags useful for testing. For instance, it sets various output -directories to point to /tmp/, and uses non-privileged ports for challenges, so -root privileges are not required. - -Activating a shell with `venv/bin/activate` sets environment variables so that -Python pulls in the correct versions of various packages needed by Certbot. -More information can be found in the `virtualenv docs`_. +After running this command, ``certbot`` and development tools like ``ipdb``, +``ipython``, ``pytest``, and ``tox`` are available in the shell where you ran +the command. These tools are installed in the virtual environment and are kept +separate from your global Python installation. This works by setting +environment variables so the right executables are found and Python can pull in +the versions of various packages needed by Certbot. More information can be +found in the `virtualenv docs`_. .. _`virtualenv docs`: https://virtualenv.pypa.io @@ -95,25 +96,24 @@ Once all the unittests pass, check for sufficient test coverage using ``tox -e cover``, and then check for code style with ``tox -e lint`` (all files) or ``pylint --rcfile=.pylintrc path/to/file.py`` (single file at a time). -Once all of the above is successful, you may run the full test suite, -including integration tests, using ``tox``. We recommend running the -commands above first, because running all tests with ``tox`` is very -slow, and the large amount of ``tox`` output can make it hard to find -specific failures when they happen. Also note that the full test suite -will attempt to modify your system's Apache config if your user has sudo -permissions, so it should not be run on a production Apache server. +Once all of the above is successful, you may run the full test suite using +``tox --skip-missing-interpreters``. We recommend running the commands above +first, because running all tests like this is very slow, and the large amount +of output can make it hard to find specific failures when they happen. -If you have trouble getting the full ``tox`` suite to run locally, it is -generally sufficient to open a pull request and let Github and Travis run -integration tests for you. +.. warning:: The full test suite may attempt to modify your system's Apache + config if your user has sudo permissions, so it should not be run on a + production Apache server. .. _integration: Integration testing with the Boulder CA ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To run integration tests locally, you need Docker and docker-compose installed -and working. Fetch and start Boulder using: +Generally it is sufficient to open a pull request and let Github and Travis run +integration tests for you, however, if you want to run them locally you need +Docker and docker-compose installed and working. Fetch and start Boulder, Let's +Encrypt's ACME CA software, by using: .. code-block:: shell @@ -316,14 +316,13 @@ Steps: 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite including coverage. The ``--skip-missing-interpreters`` argument ignores missing versions of Python needed for running the tests. Fix any errors. -5. If your code touches communication with an ACME server/Boulder, you - should run the integration tests, see `integration`_. -6. Submit the PR. -7. Did your tests pass on Travis? If they didn't, fix any errors. +5. Submit the PR. +6. Did your tests pass on Travis? If they didn't, fix any errors. Updating certbot-auto and letsencrypt-auto ========================================== + Updating the scripts -------------------- Developers should *not* modify the ``certbot-auto`` and ``letsencrypt-auto`` files @@ -386,8 +385,8 @@ Running the client with Docker ============================== You can use Docker Compose to quickly set up an environment for running and -testing Certbot. This is especially useful for macOS users. To install Docker -Compose, follow the instructions at https://docs.docker.com/compose/install/. +testing Certbot. To install Docker Compose, follow the instructions at +https://docs.docker.com/compose/install/. .. note:: Linux users can simply run ``pip install docker-compose`` to get Docker Compose after installing Docker Engine and activating your shell as @@ -420,38 +419,23 @@ OS-level dependencies can be installed like so: .. code-block:: shell - letsencrypt-auto-source/letsencrypt-auto --os-packages-only + ./certbot-auto --debug --os-packages-only In general... * ``sudo`` is required as a suggested way of running privileged process -* `Python`_ 2.7 is required +* `Python`_ 2.7 or 3.4+ is required * `Augeas`_ is required for the Python bindings -* ``virtualenv`` and ``pip`` are used for managing other python library - dependencies +* ``virtualenv`` is used for managing other Python library dependencies .. _Python: https://wiki.python.org/moin/BeginnersGuide/Download .. _Augeas: http://augeas.net/ .. _Virtualenv: https://virtualenv.pypa.io -Debian ------- - -For squeeze you will need to: - -- Use ``virtualenv --no-site-packages -p python`` instead of ``-p python2``. - - FreeBSD ------- -Packages can be installed on FreeBSD using ``pkg``, -or any other port-management tool (``portupgrade``, ``portmanager``, etc.) -from the pre-built package or can be built and installed from ports. -Either way will ensure proper installation of all the dependencies required -for the package. - FreeBSD by default uses ``tcsh``. In order to activate virtualenv (see above), you will need a compatible shell, e.g. ``pkg install bash && bash``. diff --git a/docs/install.rst b/docs/install.rst index d47264545..ead59350d 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -19,18 +19,21 @@ your system. .. _certbot.eff.org: https://certbot.eff.org +.. _system_requirements: + System Requirements =================== -Certbot currently requires Python 2.7, or 3.4+. By default, it requires -root access in order to write to ``/etc/letsencrypt``, -``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 -(if you use the ``standalone`` plugin) and to read and modify webserver -configurations (if you use the ``apache`` or ``nginx`` plugins). If none of -these apply to you, it is theoretically possible to run without root privileges, -but for most users who want to avoid running an ACME client as root, either -`letsencrypt-nosudo `_ or -`simp_le `_ are more appropriate choices. +Certbot currently requires Python 2.7 or 3.4+ running on a UNIX-like operating +system. By default, it requires root access in order to write to +``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to +bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and +modify webserver configurations (if you use the ``apache`` or ``nginx`` +plugins). If none of these apply to you, it is theoretically possible to run +without root privileges, but for most users who want to avoid running an ACME +client as root, either `letsencrypt-nosudo +`_ or `simp_le +`_ are more appropriate choices. The Apache plugin currently requires an OS with augeas version 1.0; currently `it supports From 9568f9d5b046c2114efc4ebde43f20ae31c6bdfb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 11 May 2018 10:52:45 -0700 Subject: [PATCH 31/93] Add instructions on how to ask for help (#5957) * Add instructions on how to ask for help * s/setup/set up --- docs/contributing.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index e86d1a5b3..4628c23ca 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -319,6 +319,13 @@ Steps: 5. Submit the PR. 6. Did your tests pass on Travis? If they didn't, fix any errors. +Asking for help +=============== + +If you have any questions while working on a Certbot issue, don't hesitate to +ask for help! You can do this in the #letsencrypt-dev IRC channel on Freenode. +If you don't already have an IRC client set up, we recommend you join using +`Riot `_. Updating certbot-auto and letsencrypt-auto ========================================== From 5940ee92ab5c9a9f05f7067974f6e15c9fa3205a Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 11 May 2018 14:25:02 -0700 Subject: [PATCH 32/93] add ready status type (#5941) --- acme/acme/messages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index a69b3bbc4..03dbc3255 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -145,6 +145,7 @@ STATUS_PROCESSING = Status('processing') STATUS_VALID = Status('valid') STATUS_INVALID = Status('invalid') STATUS_REVOKED = Status('revoked') +STATUS_READY = Status('ready') class IdentifierType(_Constant): From 68359086fffca8805893bf6133c53b5f75357a7f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Sun, 13 May 2018 08:06:19 -0700 Subject: [PATCH 33/93] Add link to pycon issues (#5959) * add link to pycon issues * add especially --- docs/contributing.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 4628c23ca..ba89e70fa 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -63,6 +63,10 @@ found in the `virtualenv docs`_. Find issues to work on ---------------------- +.. note:: If you're sprinting on Certbot at PyCon, you can find especially good + issues to work on during the event `here + `_. + You can find the open issues in the `github issue tracker`_. Comparatively easy ones are marked `good first issue`_. If you're starting work on something, post a comment to let others know and seek feedback on your plan From cce23c86c7fb79d883438497dd76e601f2adf687 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 14 May 2018 08:08:24 -0700 Subject: [PATCH 34/93] partially revert #5953 (#5964) --- docs/contributing.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index ba89e70fa..f152a7600 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -28,10 +28,11 @@ running: git clone https://github.com/certbot/certbot -Next you need to install dependencies and set up a virtual environment where -you can run Certbot. We recommend you do this using the commands below, -however, you can alternatively skip the rest of this section and :ref:`run -Certbot in Docker `. +If you're on macOS, we recommend you skip the rest of this section and instead +run Certbot in Docker. You can find instructions for how to do this :ref:`here +`. If you're running on Linux, you can run the following commands to +install dependencies and set up a virtual environment where you can run +Certbot. .. code-block:: shell From a0775f42baac28e5dc15206a5647c15e2c9eba8f Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 12:34:27 -0400 Subject: [PATCH 35/93] fixed issue #5969 for certbot-dns-dnsmadeeasy --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 2bac9249f..29b3ee7a6 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,3 +4,6 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True + +[mypy-certbot_dns_dnsmadeeasy.*] +check_untyped_defs = True \ No newline at end of file From 2d45b0b07ac189adf0a997b3294f1e5c0de1796e Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Mon, 14 May 2018 13:26:33 -0400 Subject: [PATCH 36/93] Check_untyped_defs in mypy with clean output for certbot_dns_rfc2136 (#5975) * check_untyped_defs in mypy with clean output for certbot_dns_rfc2136 Resolves #5973 --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 2bac9249f..01c1f7af6 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,3 +4,6 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True + +[mypy-certbot_dns_rfc2136.*] +check_untyped_defs = True From 4bd9f4dac487928131a2df67ed8227a40e4f9254 Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 13:30:38 -0400 Subject: [PATCH 37/93] fix for issue #5970 regarding certbot-dns-google --- mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 29b3ee7a6..f5ab78490 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,5 +5,5 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True -[mypy-certbot_dns_dnsmadeeasy.*] +[mypy-certbot_dns_google.*] check_untyped_defs = True \ No newline at end of file From 5636b5550749b03a23c44a652fa5dc361fd8e1a0 Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Mon, 14 May 2018 13:40:27 -0400 Subject: [PATCH 38/93] Check_untyped_defs in mypy with clean output for certbot-dns-luadns * check_untyped_defs in mypy with clean output for certbot-dns-luadns Resolves #5971 --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 01c1f7af6..135964702 100644 --- a/mypy.ini +++ b/mypy.ini @@ -7,3 +7,6 @@ check_untyped_defs = True [mypy-certbot_dns_rfc2136.*] check_untyped_defs = True + +[mypy-certbot_dns_luadns.*] +check_untyped_defs = True From 430f9414a987fc0b0f3a04edce518db170ea76a7 Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 13:42:12 -0400 Subject: [PATCH 39/93] fix for issue #5968 for certbot-dns-dnsimple --- mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index f5ab78490..79daeadd4 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,5 +5,5 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True -[mypy-certbot_dns_google.*] +[mypy-certbot_dns_dnsimple.*] check_untyped_defs = True \ No newline at end of file From 33583792fae361e6b12232c895ffdebe53a9d1b5 Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 13:43:15 -0400 Subject: [PATCH 40/93] blank line at eof --- mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 79daeadd4..a8fa8edee 100644 --- a/mypy.ini +++ b/mypy.ini @@ -6,4 +6,4 @@ ignore_missing_imports = True check_untyped_defs = True [mypy-certbot_dns_dnsimple.*] -check_untyped_defs = True \ No newline at end of file +check_untyped_defs = True From 74448e934491f7d8e731a5473febe1dc54dd1d15 Mon Sep 17 00:00:00 2001 From: Kevin Le Date: Mon, 14 May 2018 13:45:12 -0400 Subject: [PATCH 41/93] Set pause=False to fix view_config_changes (#5977) --- certbot/reverter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/reverter.py b/certbot/reverter.py index 34feafc7e..15ad1a987 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -181,7 +181,7 @@ class Reverter(object): if for_logging: return os.linesep.join(output) zope.component.getUtility(interfaces.IDisplay).notification( - os.linesep.join(output), force_interactive=True) + os.linesep.join(output), force_interactive=True, pause=False) def add_to_temp_checkpoint(self, save_files, save_notes): """Add files to temporary checkpoint. From be03a976d5a649a24810a85bb4f86d7caae95ddd Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 13:54:26 -0400 Subject: [PATCH 42/93] fixed issue #5969 for certbot-dns-dnsmadeeasy (#5976) --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 01c1f7af6..6580de6de 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,5 +5,8 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True +[mypy-certbot_dns_dnsmadeeasy.*] +check_untyped_defs = True + [mypy-certbot_dns_rfc2136.*] check_untyped_defs = True From c1471fe873e4b72a01638e75b2f30bb111c3bf92 Mon Sep 17 00:00:00 2001 From: Kevin Le Date: Mon, 14 May 2018 16:15:02 -0400 Subject: [PATCH 43/93] Document IPv6/IPv4 binding on standalone. (#5983) --- docs/using.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/using.rst b/docs/using.rst index 7a25a5cc2..272f5ac6e 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -170,6 +170,14 @@ one of the options shown below on the command line. It must still be possible for your machine to accept inbound connections from the Internet on the specified port using each requested domain name. +By default, Certbot first attempts to bind to the port for all interfaces using +IPv6 and then bind to that port using IPv4; Certbot continues so long as at +least one bind succeeds. On most Linux systems, IPv4 traffic will be routed to +the bound IPv6 port and the failure during the second bind is expected. + +Use ``---address`` to explicitly tell Certbot which interface +(and protocol) to bind. + .. note:: The ``--standalone-supported-challenges`` option has been deprecated since ``certbot`` version 0.9.0. From 907ee797151f270bec3a2697743568362db497cd Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Mon, 14 May 2018 16:18:49 -0400 Subject: [PATCH 44/93] Check_untyped_defs in mypy with clean output for certbot-dns-nsone (#5987) * check_untyped_defs in mypy with clean output for certbot-dns-nsone Resolves #5972 --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 8f1bc91ef..449262795 100644 --- a/mypy.ini +++ b/mypy.ini @@ -17,5 +17,8 @@ check_untyped_defs = True [mypy-certbot_dns_luadns.*] check_untyped_defs = True +[mypy-certbot_dns_nsone.*] +check_untyped_defs = True + [mypy-certbot_dns_rfc2136.*] check_untyped_defs = True From a724dc659ba7590dfae21fd6bfdf6c0a13766283 Mon Sep 17 00:00:00 2001 From: Sarah Braden Date: Mon, 14 May 2018 17:21:09 -0400 Subject: [PATCH 45/93] correct error message text now prompts user to run $certbot certificates (#5988) --- certbot/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/main.py b/certbot/main.py index a041b998f..0ae5b9d7a 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -324,7 +324,7 @@ def _find_lineage_for_domains_and_certname(config, domains, certname): return "newcert", None else: raise errors.ConfigurationError("No certificate with name {0} found. " - "Use -d to specify domains, or run certbot --certificates to see " + "Use -d to specify domains, or run certbot certificates to see " "possible certificate names.".format(certname)) def _get_added_removed(after, before): From 372d4a046cb5b3f937469d4220c0fb6ba4a4f85c Mon Sep 17 00:00:00 2001 From: speter Date: Mon, 14 May 2018 23:40:06 +0200 Subject: [PATCH 46/93] docs/using.rst: fix typo (#5962) --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 272f5ac6e..40d8f8452 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -609,7 +609,7 @@ commands into your individual environment. .. note:: ``certbot renew`` exit status will only be 1 if a renewal attempt failed. This means ``certbot renew`` exit status will be 0 if no certificate needs to be updated. If you write a custom script and expect to run a command only after a certificate was actually renewed - you will need to use the ``--post-hook`` since the exit status will be 0 both on successful renewal + you will need to use the ``--deploy-hook`` since the exit status will be 0 both on successful renewal and when renewal is not necessary. .. _renewal-config-file: From 02b128a12831f4c43fb734dd1e1ca08deb66142c Mon Sep 17 00:00:00 2001 From: signop <39252060+signop@users.noreply.github.com> Date: Mon, 14 May 2018 14:43:43 -0700 Subject: [PATCH 47/93] Add support for specifying source_address to ClientNetwork. (#5990) For certbot/certbot#3489 --- acme/acme/client.py | 14 +++++++++++++- acme/acme/client_test.py | 25 +++++++++++++++++++++++++ acme/setup.py | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 7807f0ece..d4f5a4787 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -12,7 +12,9 @@ from six.moves import http_client # pylint: disable=import-error import josepy as jose import OpenSSL import re +from requests_toolbelt.adapters.source import SourceAddressAdapter import requests +from requests.adapters import HTTPAdapter import sys from acme import crypto_util @@ -857,9 +859,12 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes :param bool verify_ssl: Whether to verify certificates on SSL connections. :param str user_agent: String to send as User-Agent header. :param float timeout: Timeout for requests. + :param source_address: Optional source address to bind to when making requests. + :type source_address: str or tuple(str, int) """ def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True, - user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT): + user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT, + source_address=None): # pylint: disable=too-many-arguments self.key = key self.account = account @@ -869,6 +874,13 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes self.user_agent = user_agent self.session = requests.Session() self._default_timeout = timeout + adapter = HTTPAdapter() + + if source_address is not None: + adapter = SourceAddressAdapter(source_address) + + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) def __del__(self): # Try to close the session, but don't show exceptions to the diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index c17b83210..f3018ed81 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -1129,6 +1129,31 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.assertRaises(requests.exceptions.RequestException, self.net.post, 'uri', obj=self.obj) +class ClientNetworkSourceAddressBindingTest(unittest.TestCase): + """Tests that if ClientNetwork has a source IP set manually, the underlying library has + used the provided source address.""" + + def setUp(self): + self.source_address = "8.8.8.8" + + def test_source_address_set(self): + from acme.client import ClientNetwork + net = ClientNetwork(key=None, alg=None, source_address=self.source_address) + for adapter in net.session.adapters.values(): + self.assertTrue(self.source_address in adapter.source_address) + + def test_behavior_assumption(self): + """This is a test that guardrails the HTTPAdapter behavior so that if the default for + a Session() changes, the assumptions here aren't violated silently.""" + from acme.client import ClientNetwork + # Source address not specified, so the default adapter type should be bound -- this + # test should fail if the default adapter type is changed by requests + net = ClientNetwork(key=None, alg=None) + session = requests.Session() + for scheme in session.adapters.keys(): + client_network_adapter = net.session.adapters.get(scheme) + default_adapter = session.adapters.get(scheme) + self.assertEqual(client_network_adapter.__class__, default_adapter.__class__) if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/setup.py b/acme/setup.py index 72ab5919b..e91c36b3d 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -19,6 +19,7 @@ install_requires = [ 'pyrfc3339', 'pytz', 'requests[security]>=2.4.1', # security extras added in 2.4.1 + 'requests-toolbelt>=0.3.0', 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible ] From 99d94cc7e8311aaedccbdc9445d8bc7323ededde Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Mon, 14 May 2018 17:45:25 -0400 Subject: [PATCH 48/93] Make request logs pretty in Python 3 (#5992) Decode response data as UTF-8 to eliminate ugly bytes repr in Python 3. Resolves #5932 --- acme/acme/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index d4f5a4787..bdc07fb1c 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1030,7 +1030,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes if response.headers.get("Content-Type") == DER_CONTENT_TYPE: debug_content = base64.b64encode(response.content) else: - debug_content = response.content + debug_content = response.content.decode("utf-8") logger.debug('Received response:\nHTTP %d\n%s\n\n%s', response.status_code, "\n".join(["{0}: {1}".format(k, v) From 42ef2520432fa8b40307e612ad3de8185af5ffc2 Mon Sep 17 00:00:00 2001 From: James Hiebert Date: Tue, 15 May 2018 10:58:33 -0400 Subject: [PATCH 49/93] Adds a note about Python3 in the Developer Guide (#5998) --- docs/contributing.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index f152a7600..52f08efe0 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -40,6 +40,12 @@ Certbot. ./certbot-auto --debug --os-packages-only tools/venv.sh +If you have Python3 available and want to use it, run the ``venv3.sh`` script. + +.. code-block:: shell + + tools/venv3.sh + .. note:: You may need to repeat this when Certbot's dependencies change or when a new plugin is introduced. @@ -50,6 +56,8 @@ latter by running: .. code-block:: shell source venv/bin/activate + # or + source venv3/bin/activate After running this command, ``certbot`` and development tools like ``ipdb``, ``ipython``, ``pytest``, and ``tox`` are available in the shell where you ran From 2d68c9b81e2ed437edc7a9175562be5c246d4929 Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Tue, 15 May 2018 11:46:36 -0400 Subject: [PATCH 50/93] Display (None) instead of a bullet for empty lists (#5999) Include a line break before "(None)" to maintain consistency with output for lists that are not empty. Previous result as expected for non-empty lists: >>> _format_list('+', ['one', 'two', 'three']) '\n+ one\n+ two\n+ three' Previous unexpected result for empty lists: >>> _format_list('+', []) '\n+ ' New result as expected (unchanged) for non-empty lists: >>> _format_list('+', ['one', 'two', 'three']) '\n+ one\n+ two\n+ three' New behavior more explicit for empty lists: >>> _format_list('+', []) '\n(None)' Resolves #5886 --- certbot/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/certbot/main.py b/certbot/main.py index 0ae5b9d7a..8a9a37084 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -340,7 +340,10 @@ def _get_added_removed(after, before): def _format_list(character, strings): """Format list with given character """ - formatted = "{br}{ch} " + "{br}{ch} ".join(strings) + if len(strings) == 0: + formatted = "{br}(None)" + else: + formatted = "{br}{ch} " + "{br}{ch} ".join(strings) return formatted.format( ch=character, br=os.linesep From 802fcc99ee2150989e5a1e860b6aad490691d72f Mon Sep 17 00:00:00 2001 From: signop <39252060+signop@users.noreply.github.com> Date: Tue, 15 May 2018 08:50:09 -0700 Subject: [PATCH 51/93] Add requests-toolbelt hashes to requirements. (#6001) Fixes certbot/certbot#5993 --- letsencrypt-auto-source/letsencrypt-auto | 3 +++ letsencrypt-auto-source/pieces/dependency-requirements.txt | 3 +++ 2 files changed, 6 insertions(+) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 853c66023..0b83b08a7 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1187,6 +1187,9 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +requests-toolbelt==0.8.0 \ + --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ + --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 # Contains the requirements for the letsencrypt package. # diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 1e69af9c2..a30a32b48 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -191,3 +191,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +requests-toolbelt==0.8.0 \ + --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \ + --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5 From 307f45f88f09872e5e9f5c1e7f2bae4361a3096d Mon Sep 17 00:00:00 2001 From: James Hiebert Date: Tue, 15 May 2018 12:36:47 -0400 Subject: [PATCH 52/93] Enable checking of type annotation in Nginx plugin (#5997) * Adds type checking for certbot-nginx * First pass at type annotation in certbot-nginx * Ensure linting is disabled for timing imports * Makes container types specific per PR comments * Removes unnecessary lint option --- certbot-nginx/certbot_nginx/configurator.py | 13 +++++--- certbot-nginx/certbot_nginx/http_01.py | 3 +- certbot-nginx/certbot_nginx/nginxparser.py | 2 +- certbot-nginx/certbot_nginx/parser.py | 31 +++++++++++-------- .../certbot_nginx/tests/parser_test.py | 3 +- mypy.ini | 3 ++ 6 files changed, 34 insertions(+), 21 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 378f24b63..118699aa2 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -28,6 +28,9 @@ from certbot_nginx import nginxparser from certbot_nginx import parser from certbot_nginx import tls_sni_01 from certbot_nginx import http_01 +from certbot_nginx import obj # pylint: disable=unused-import +from acme.magic_typing import List, Dict, Set # pylint: disable=unused-import, no-name-in-module + logger = logging.getLogger(__name__) @@ -98,8 +101,8 @@ class NginxConfigurator(common.Installer): # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = {} - self._wildcard_redirect_vhosts = {} + self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_redirect_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] # Add number of outstanding challenges self._chall_out = 0 @@ -528,7 +531,7 @@ class NginxConfigurator(common.Installer): :rtype: set """ - all_names = set() + all_names = set() # type: Set[str] for vhost in self.parser.get_vhosts(): all_names.update(vhost.names) @@ -824,7 +827,7 @@ class NginxConfigurator(common.Installer): self.parser.add_server_directives(vhost, stapling_directives) except errors.MisconfigurationError as error: - logger.debug(error) + logger.debug(str(error)) raise errors.PluginError("An error occurred while enabling OCSP " "stapling for {0}.".format(vhost.names)) @@ -892,7 +895,7 @@ class NginxConfigurator(common.Installer): universal_newlines=True) text = proc.communicate()[1] # nginx prints output to stderr except (OSError, ValueError) as error: - logger.debug(error, exc_info=True) + logger.debug(str(error), exc_info=True) raise errors.PluginError( "Unable to run %s -V" % self.conf('ctl')) diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index d08a3b1cb..677ce0737 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -10,6 +10,7 @@ from certbot.plugins import common from certbot_nginx import obj from certbot_nginx import nginxparser +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module logger = logging.getLogger(__name__) @@ -113,7 +114,7 @@ class NginxHttp01(common.ChallengePerformer): :returns: list of :class:`certbot_nginx.obj.Addr` to apply :rtype: list """ - addresses = [] + addresses = [] # type: List[obj.Addr] default_addr = "%s" % self.configurator.config.http01_port ipv6_addr = "[::]:{0}".format( self.configurator.config.http01_port) diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index 14481e298..8818bc040 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -248,7 +248,7 @@ class UnspacedList(list): """Recurse through the parse tree to figure out if any sublists are dirty""" if self.dirty: return True - return any((isinstance(x, list) and x.is_dirty() for x in self)) + return any((isinstance(x, UnspacedList) and x.is_dirty() for x in self)) def _spaced_position(self, idx): "Convert from indexes in the unspaced list to positions in the spaced one" diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index f06cd17a7..5bc7946dc 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -13,7 +13,7 @@ from certbot import errors from certbot_nginx import obj from certbot_nginx import nginxparser - +from acme.magic_typing import Union, Dict, Set, Any, List, Tuple # pylint: disable=unused-import, no-name-in-module logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ class NginxParser(object): """ def __init__(self, root): - self.parsed = {} + self.parsed = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] self.root = os.path.abspath(root) self.config_root = self._find_config_root() @@ -90,7 +90,7 @@ class NginxParser(object): """ servers = self._get_raw_servers() - addr_to_ssl = {} + addr_to_ssl = {} # type: Dict[Tuple[str, str], bool] for filename in servers: for server, _ in servers[filename]: # Parse the server block to save addr info @@ -104,9 +104,10 @@ class NginxParser(object): def _get_raw_servers(self): # pylint: disable=cell-var-from-loop + # type: () -> Dict """Get a map of unparsed all server blocks """ - servers = {} + servers = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] for filename in self.parsed: tree = self.parsed[filename] servers[filename] = [] @@ -727,9 +728,9 @@ def _parse_server_raw(server): :rtype: dict """ - parsed_server = {'addrs': set(), - 'ssl': False, - 'names': set()} + addrs = set() # type: Set[obj.Addr] + ssl = False # type: bool + names = set() # type: Set[str] apply_ssl_to_all_addrs = False @@ -739,17 +740,21 @@ def _parse_server_raw(server): if directive[0] == 'listen': addr = obj.Addr.fromstring(" ".join(directive[1:])) if addr: - parsed_server['addrs'].add(addr) + addrs.add(addr) if addr.ssl: - parsed_server['ssl'] = True + ssl = True elif directive[0] == 'server_name': - parsed_server['names'].update(x.strip('"\'') for x in directive[1:]) + names.update(x.strip('"\'') for x in directive[1:]) elif _is_ssl_on_directive(directive): - parsed_server['ssl'] = True + ssl = True apply_ssl_to_all_addrs = True if apply_ssl_to_all_addrs: - for addr in parsed_server['addrs']: + for addr in addrs: addr.ssl = True - return parsed_server + return { + 'addrs': addrs, + 'ssl': ssl, + 'names': names + } diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 1e9703185..5a37c9565 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -11,6 +11,7 @@ from certbot_nginx import nginxparser from certbot_nginx import obj from certbot_nginx import parser from certbot_nginx.tests import util +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods @@ -99,7 +100,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ([[[0], [3], [4]], [[5], [3], [0]]], [])] for mylist, result in mylists: - paths = [] + paths = [] # type: List[List[int]] parser._do_for_subarray(mylist, lambda x: isinstance(x, list) and len(x) >= 1 and diff --git a/mypy.ini b/mypy.ini index 449262795..9709fa5ed 100644 --- a/mypy.ini +++ b/mypy.ini @@ -22,3 +22,6 @@ check_untyped_defs = True [mypy-certbot_dns_rfc2136.*] check_untyped_defs = True + +[mypy-certbot_nginx.*] +check_untyped_defs = True From 9bd5b3dda2ca606614892cb178feeab8efe9a1f9 Mon Sep 17 00:00:00 2001 From: dschlessman <36820111+dschlessman@users.noreply.github.com> Date: Tue, 15 May 2018 13:40:32 -0400 Subject: [PATCH 53/93] Issue 5951/check untyped defs apache (#5989) * resolved mypy untyped defs in parser.py * resolved mypy untyped defs in obj.py * removed unused imports * resolved mypy untyped defs in http_01.py * resolved mypy untyped defs in tls_sni_01.py * resolved mypy untyped defs in configurator.py * address mypy too-many-arguments error in override_centos.py * resolved mypy untyped defs in http_01_test.py * removed unused 'conf' argument that was causing mypy method assignment error * address mypy error where same variable reassigned to different type * address pylint and coverage issues * one character space change for formatting * fix required acme version for certbot-apache --- certbot-apache/certbot_apache/configurator.py | 24 ++++++++++--------- certbot-apache/certbot_apache/http_01.py | 5 ++-- certbot-apache/certbot_apache/obj.py | 5 ++-- .../certbot_apache/override_centos.py | 4 ++-- certbot-apache/certbot_apache/parser.py | 11 +++++---- .../certbot_apache/tests/configurator_test.py | 9 +++---- .../certbot_apache/tests/http_01_test.py | 4 ++-- certbot-apache/certbot_apache/tests/util.py | 5 ---- certbot-apache/certbot_apache/tls_sni_01.py | 3 ++- certbot-apache/local-oldest-requirements.txt | 2 +- certbot-apache/setup.py | 2 +- mypy.ini | 3 +++ 12 files changed, 39 insertions(+), 38 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 03ba05bb0..bb82a9d3f 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -13,11 +13,13 @@ import zope.component import zope.interface from acme import challenges +from acme.magic_typing import DefaultDict, Dict, List, Set # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot import util +from certbot.achallenges import KeyAuthorizationAnnotatedChallenge # pylint: disable=unused-import from certbot.plugins import common from certbot.plugins.util import path_surgery @@ -150,14 +152,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): super(ApacheConfigurator, self).__init__(*args, **kwargs) # Add name_server association dict - self.assoc = dict() + self.assoc = dict() # type: Dict[str, obj.VirtualHost] # Outstanding challenges - self._chall_out = set() + self._chall_out = set() # type: Set[KeyAuthorizationAnnotatedChallenge] # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = dict() + self._wildcard_vhosts = dict() # type: Dict[str, List[obj.VirtualHost]] # Maps enhancements to vhosts we've enabled the enhancement for - self._enhanced_vhosts = defaultdict(set) + self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] # These will be set in the prepare function self.parser = None @@ -659,7 +661,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: set """ - all_names = set() + all_names = set() # type: Set[str] vhost_macro = [] @@ -800,8 +802,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Search base config, and all included paths for VirtualHosts - file_paths = {} - internal_paths = defaultdict(set) + file_paths = {} # type: Dict[str, str] + internal_paths = defaultdict(set) # type: DefaultDict[str, Set[str]] vhs = [] # Make a list of parser paths because the parser_paths # dictionary may be modified during the loop. @@ -1239,7 +1241,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if not self.parser.parsed_in_current(ssl_fp): self.parser.parse_file(ssl_fp) except IOError: - logger.fatal("Error writing/reading to file in make_vhost_ssl") + logger.critical("Error writing/reading to file in make_vhost_ssl", exc_info=True) raise errors.PluginError("Unable to write/read in make_vhost_ssl") if sift: @@ -1327,7 +1329,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): try: span_val = self.aug.span(vhost.path) except ValueError: - logger.fatal("Error while reading the VirtualHost %s from " + logger.critical("Error while reading the VirtualHost %s from " "file %s", vhost.name, vhost.filep, exc_info=True) raise errors.PluginError("Unable to read VirtualHost from file") span_filep = span_val[0] @@ -1770,7 +1772,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # There can be other RewriteRule directive lines in vhost config. # rewrite_args_dict keys are directive ids and the corresponding value # for each is a list of arguments to that directive. - rewrite_args_dict = defaultdict(list) + rewrite_args_dict = defaultdict(list) # type: DefaultDict[str, List[str]] pat = r'(.*directive\[\d+\]).*' for match in rewrite_path: m = re.match(pat, match) @@ -1864,7 +1866,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if ssl_vhost.aliases: serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases) - rewrite_rule_args = [] + rewrite_rule_args = [] # type: List[str] if self.get_version() >= (2, 3, 9): rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END else: diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index cce93a646..37545e9cc 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -2,9 +2,10 @@ import logging import os +from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module from certbot import errors - from certbot.plugins import common +from certbot_apache.obj import VirtualHost # pylint: disable=unused-import logger = logging.getLogger(__name__) @@ -51,7 +52,7 @@ class ApacheHttp01(common.TLSSNI01): self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") - self.moded_vhosts = set() + self.moded_vhosts = set() # type: Set[VirtualHost] def perform(self): """Perform all HTTP-01 challenges.""" diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index fcf3bfe08..290979f27 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -1,6 +1,7 @@ """Module contains classes used by the Apache Configurator.""" import re +from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module from certbot.plugins import common @@ -140,7 +141,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def get_names(self): """Return a set of all names.""" - all_names = set() + all_names = set() # type: Set[str] all_names.update(self.aliases) # Strip out any scheme:// and field from servername if self.name is not None: @@ -251,7 +252,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods # already_found acts to keep everything very conservative. # Don't allow multiple ip:ports in same set. - already_found = set() + already_found = set() # type: Set[str] for addr in vhost.addrs: for local_addr in self.addrs: diff --git a/certbot-apache/certbot_apache/override_centos.py b/certbot-apache/certbot_apache/override_centos.py index 6e75e361d..0b6b12b96 100644 --- a/certbot-apache/certbot_apache/override_centos.py +++ b/certbot-apache/certbot_apache/override_centos.py @@ -47,10 +47,10 @@ class CentOSParser(parser.ApacheParser): self.sysconfig_filep = "/etc/sysconfig/httpd" super(CentOSParser, self).__init__(*args, **kwargs) - def update_runtime_variables(self, *args, **kwargs): + def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ # Opportunistic, works if SELinux not enforced - super(CentOSParser, self).update_runtime_variables(*args, **kwargs) + super(CentOSParser, self).update_runtime_variables() self.parse_sysconfig_var() def parse_sysconfig_var(self): diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index d7da1e55e..43878eda2 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -9,6 +9,7 @@ import sys import six +from acme.magic_typing import Dict, List, Set # pylint: disable=unused-import, no-name-in-module from certbot import errors logger = logging.getLogger(__name__) @@ -38,9 +39,9 @@ class ApacheParser(object): # issues with aug.load() after adding new files / defines to parse tree self.configurator = configurator - self.modules = set() - self.parser_paths = {} - self.variables = {} + self.modules = set() # type: Set[str] + self.parser_paths = {} # type: Dict[str, List[str]] + self.variables = {} # type: Dict[str, str] self.aug = aug # Find configuration root and make sure augeas can parse it. @@ -119,7 +120,7 @@ class ApacheParser(object): the iteration issue. Else... parse and enable mods at same time. """ - mods = set() + mods = set() # type: Set[str] matches = self.find_dir("LoadModule") iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration @@ -408,7 +409,7 @@ class ApacheParser(object): else: arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg) - ordered_matches = [] + ordered_matches = [] # type: List[str] # TODO: Wildcards should be included in alphabetical order # https://httpd.apache.org/docs/2.4/mod/core.html#include diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index e33e16843..23c1ee82b 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -353,14 +353,11 @@ class MultipleVhostsTest(util.ApacheTest): self.config.parser.find_dir = mock_find_dir mock_add.reset_mock() - self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access - tried_to_add = [] for a in mock_add.call_args_list: - tried_to_add.append(a[0][1] == "Include" and - a[0][2] == self.config.mod_ssl_conf) - # Include shouldn't be added, as patched find_dir "finds" existing one - self.assertFalse(any(tried_to_add)) + if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf: + self.fail("Include shouldn't be added, as patched find_dir 'finds' existing one") \ + # pragma: no cover def test_deploy_cert(self): self.config.parser.modules.add("ssl_module") diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index dc1ca34d6..98bf412ae 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -4,12 +4,12 @@ import os import unittest from acme import challenges +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot.tests import acme_util - from certbot_apache.tests import util @@ -23,7 +23,7 @@ class ApacheHttp01Test(util.ApacheTest): super(ApacheHttp01Test, self).setUp(*args, **kwargs) self.account_key = self.rsa512jwk - self.achalls = [] + self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") # Takes the vhosts for encryption-example.demo, certbot.demo, and diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 1daaa00c5..6d3cfa109 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -87,7 +87,6 @@ class ParserTest(ApacheTest): def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-locals config_path, vhost_path, config_dir, work_dir, version=(2, 4, 7), - conf=None, os_info="generic", conf_vhost_path=None): """Create an Apache Configurator with the specified options. @@ -133,10 +132,6 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc config_class = configurator.ApacheConfigurator config = config_class(config=mock_le_config, name="apache", version=version) - # This allows testing scripts to set it a bit more - # quickly - if conf is not None: - config.conf = conf # pragma: no cover config.prepare() return config diff --git a/certbot-apache/certbot_apache/tls_sni_01.py b/certbot-apache/certbot_apache/tls_sni_01.py index 549feb17d..65230cdcb 100644 --- a/certbot-apache/certbot_apache/tls_sni_01.py +++ b/certbot-apache/certbot_apache/tls_sni_01.py @@ -3,6 +3,7 @@ import os import logging +from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module from certbot.plugins import common from certbot.errors import PluginError, MissingCommandlineFlag @@ -93,7 +94,7 @@ class ApacheTlsSni01(common.TLSSNI01): :rtype: set """ - addrs = set() + addrs = set() # type: Set[obj.Addr] config_text = "\n" for achall in self.achalls: diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt index 8368d266e..724b61d3f 100644 --- a/certbot-apache/local-oldest-requirements.txt +++ b/certbot-apache/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 +-e acme[dev] certbot[dev]==0.21.1 diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f0c20da7c..0e4304300 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -9,7 +9,7 @@ version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', + 'acme>0.24.0', 'certbot>=0.21.1', 'mock', 'python-augeas', diff --git a/mypy.ini b/mypy.ini index 9709fa5ed..7e08a1f99 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,9 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True +[mypy-certbot_apache.*] +check_untyped_defs = True + [mypy-certbot_dns_dnsimple.*] check_untyped_defs = True From 751f9843b482f276bbe6975eaa52be86059833e6 Mon Sep 17 00:00:00 2001 From: GmH Date: Tue, 15 May 2018 14:22:09 -0400 Subject: [PATCH 54/93] fixed issue #5974 for certbot-dns-route53 (#5984) * fixed issue #5974 for certbot-dns-route53 * fixed issue #5967 for certbot-dns-digitalocean * update to use acme.magic_typing and DefaultDict class * added no-name-in-module identifier, for issue #5974 * added unused-import identifier to disable option, for issue #5974 --- certbot-dns-route53/certbot_dns_route53/dns_route53.py | 4 +++- certbot-dns-route53/local-oldest-requirements.txt | 2 +- certbot-dns-route53/setup.py | 2 +- mypy.ini | 6 ++++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index 27d185656..f71935de2 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -11,6 +11,8 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from acme.magic_typing import DefaultDict, List, Dict # pylint: disable=unused-import, no-name-in-module + logger = logging.getLogger(__name__) INSTRUCTIONS = ( @@ -34,7 +36,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) self.r53 = boto3.client("route53") - self._resource_records = collections.defaultdict(list) + self._resource_records = collections.defaultdict(list) # type: DefaultDict[str, List[Dict[str, str]]] def more_info(self): # pylint: disable=missing-docstring,no-self-use return "Solve a DNS01 challenge using AWS Route53" diff --git a/certbot-dns-route53/local-oldest-requirements.txt b/certbot-dns-route53/local-oldest-requirements.txt index 8368d266e..724b61d3f 100644 --- a/certbot-dns-route53/local-oldest-requirements.txt +++ b/certbot-dns-route53/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 +-e acme[dev] certbot[dev]==0.21.1 diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 84b701638..083cd15ae 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -8,7 +8,7 @@ version = '0.25.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', + 'acme>0.24.0', 'certbot>=0.21.1', 'boto3', 'mock', diff --git a/mypy.ini b/mypy.ini index 7e08a1f99..506c253a8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -26,5 +26,11 @@ check_untyped_defs = True [mypy-certbot_dns_rfc2136.*] check_untyped_defs = True +[mypy-certbot_dns_route53.*] +check_untyped_defs = True + +[mypy-certbot_dns_digitalocean.*] +check_untyped_defs = True + [mypy-certbot_nginx.*] check_untyped_defs = True From 722dac86d558af58c1aed27ad9d2b71bfee946ae Mon Sep 17 00:00:00 2001 From: Douglas Anger Date: Tue, 15 May 2018 15:50:47 -0400 Subject: [PATCH 55/93] Fix crash when email submission endpoint unavailable (#6002) * Fix crash when email submission endpoint unavailable Handle KeyError and ValueError so that if the email submission endpoint goes down, Certbot can still run. Add tests to eff_test.py: - simulate non-JSON response as described in issue #5858 - simulate JSON response without 'status' element Non-JSON response throws an uncaught ValueError when attempting to decode as JSON. A JSON response missing the 'status' element throws an uncaught KeyError when checking whether status is True or False. Teach _check_response to handle ValueError and KeyError and report an issue to the user. Rewrite if statement as assertion with try-except block to make error handling consistent within the function. Update test_not_ok to make mocked raise_for_status function raise a requests.exceptions.HTTPError. Resolves #5858 * Update PR with requested changes - Use `if` instead of `assert` to check `status` element of response JSON - Handle KeyError and ValueError in the same way - Import requests at the beginning of eff_test.py - Clear JSON in test case in a more idiomatic way --- certbot/eff.py | 9 ++++++--- certbot/tests/eff_test.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/certbot/eff.py b/certbot/eff.py index 746261faa..b047c0b97 100644 --- a/certbot/eff.py +++ b/certbot/eff.py @@ -71,11 +71,14 @@ def _check_response(response): """ logger.debug('Received response:\n%s', response.content) - if response.ok: - if not response.json()['status']: + try: + response.raise_for_status() + if response.json()['status'] == False: _report_failure('your e-mail address appears to be invalid') - else: + except requests.exceptions.HTTPError: _report_failure() + except (ValueError, KeyError): + _report_failure('there was a problem with the server response') def _report_failure(reason=None): diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index 160af1993..8d0d5778c 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -1,4 +1,5 @@ """Tests for certbot.eff.""" +import requests import unittest import mock @@ -118,11 +119,28 @@ class SubscribeTest(unittest.TestCase): @test_util.patch_get_utility() def test_not_ok(self, mock_get_utility): self.response.ok = False + self.response.raise_for_status.side_effect = requests.exceptions.HTTPError self._call() # pylint: disable=no-value-for-parameter actual = self._get_reported_message(mock_get_utility) unexpected_part = 'because' self.assertFalse(unexpected_part in actual) + @test_util.patch_get_utility() + def test_response_not_json(self, mock_get_utility): + self.response.json.side_effect = ValueError() + self._call() # pylint: disable=no-value-for-parameter + actual = self._get_reported_message(mock_get_utility) + expected_part = 'problem' + self.assertTrue(expected_part in actual) + + @test_util.patch_get_utility() + def test_response_json_missing_status_element(self, mock_get_utility): + self.json.clear() + self._call() # pylint: disable=no-value-for-parameter + actual = self._get_reported_message(mock_get_utility) + expected_part = 'problem' + self.assertTrue(expected_part in actual) + def _get_reported_message(self, mock_get_utility): self.assertTrue(mock_get_utility().add_message.called) return mock_get_utility().add_message.call_args[0][0] From 41e1976c178dac8875584e8f11d68e28515e0a01 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 16 May 2018 06:24:14 -0700 Subject: [PATCH 56/93] Fix noisy tests (#6004) * Fixes #5570. The issue is calls to atexit aren't mocked out. During the tests there are many repeated calls registering functions to be called when the process exits so when the tests finishes, it prints a ton of output from running those registered functions. This suppresses that by mocking out atexit. * Mock at a lower level. This ensures we don't mess with any other mocks in this test class by mocking at the lowest level we can. Other tests shouldn't be mocking out specific internals of functions in other modules, so this should work just fine. --- certbot/tests/main_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 22653ca3a..0986ff060 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -626,7 +626,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met toy_stdout = stdout if stdout else six.StringIO() with mock.patch('certbot.main.sys.stdout', new=toy_stdout): with mock.patch('certbot.main.sys.stderr') as stderr: - ret = main.main(args[:]) # NOTE: parser can alter its args! + with mock.patch("certbot.util.atexit"): + ret = main.main(args[:]) # NOTE: parser can alter its args! return ret, toy_stdout, stderr def test_no_flags(self): From 20418cdd6899b0ac5c0e7669bdc69ed135594276 Mon Sep 17 00:00:00 2001 From: pdamodaran Date: Thu, 17 May 2018 09:52:11 -0400 Subject: [PATCH 57/93] Fixed #5859 (#6011) --- tests/boulder-fetch.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index fc9cbaae7..d513ec064 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -17,12 +17,6 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml -# If we're testing against ACMEv2, we need to use a newer boulder config for -# now. See https://github.com/letsencrypt/boulder#quickstart. -if [ "$BOULDER_INTEGRATION" = "v2" ]; then - sed -i 's/BOULDER_CONFIG_DIR: .*/BOULDER_CONFIG_DIR: test\/config-next/' docker-compose.yml -fi - docker-compose up -d set +x # reduce verbosity while waiting for boulder From 1be1bd92115101e019d71a23ee14e7bfadf72226 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 17 May 2018 09:23:05 -0700 Subject: [PATCH 58/93] remove PYTHONPATH (#6016) --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 38d4e6ae1..8c4d6c38d 100644 --- a/tox.ini +++ b/tox.ini @@ -58,7 +58,6 @@ commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py setenv = - PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 [testenv:py27-oldest] From 9b2862ebb0f1618161fcf5726ac4de812b959ac9 Mon Sep 17 00:00:00 2001 From: TyrannosourceExe <38411281+TyrannosourceExe@users.noreply.github.com> Date: Thu, 17 May 2018 19:03:01 -0400 Subject: [PATCH 59/93] 3692 --dry-run expiration emails (#6015) * If --dry-run is used and there exists no staging account, create account with no email * added unit testing of dry-run to ensure certbot does not ask the user to create an email, and that certbot creates an account with no email --- certbot/client.py | 4 ++++ certbot/tests/client_test.py | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 45dc9c63b..cdfbdc252 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -155,6 +155,10 @@ def register(config, account_storage, tos_cb=None): if not config.dry_run: logger.info("Registering without email!") + # If --dry-run is used, and there is no staging account, create one with no email. + if config.dry_run: + config.email = None + # Each new registration shall use a fresh new key key = jose.JWKRSA(key=jose.ComparableRSAKey( rsa.generate_private_key( diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 6add141d4..a4425bca9 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -12,7 +12,6 @@ from certbot import util import certbot.tests.util as test_util - KEY = test_util.load_vector("rsa512_key.pem") CSR_SAN = test_util.load_vector("csr-san_512.pem") @@ -92,6 +91,20 @@ class RegisterTest(test_util.ConfigTestCase): mock_logger.info.assert_called_once_with(mock.ANY) self.assertTrue(mock_handle.called) + @mock.patch("certbot.account.report_new_account") + @mock.patch("certbot.client.display_ops.get_email") + def test_dry_run_no_staging_account(self, _rep, mock_get_email): + """Tests dry-run for no staging account, expect account created with no email""" + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + with mock.patch("certbot.eff.handle_subscription"): + with mock.patch("certbot.account.report_new_account"): + self.config.dry_run = True + self._call() + # check Certbot did not ask the user to provide an email + self.assertFalse(mock_get_email.called) + # check Certbot created an account with no email. Contact should return empty + self.assertFalse(mock_client().new_account_and_tos.call_args[0][0].contact) + def test_unsupported_error(self): from acme import messages msg = "Test" @@ -105,6 +118,7 @@ class RegisterTest(test_util.ConfigTestCase): class ClientTestCommon(test_util.ConfigTestCase): """Common base class for certbot.client.Client tests.""" + def setUp(self): super(ClientTestCommon, self).setUp() self.config.no_verify_ssl = False @@ -124,6 +138,7 @@ class ClientTestCommon(test_util.ConfigTestCase): class ClientTest(ClientTestCommon): """Tests for certbot.client.Client.""" + def setUp(self): super(ClientTest, self).setUp() @@ -286,10 +301,10 @@ class ClientTest(ClientTestCommon): @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') def test_obtain_and_enroll_certificate(self, - mock_storage, mock_obtain_certificate): + mock_storage, mock_obtain_certificate): domains = ["*.example.com", "example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), - mock.MagicMock(), mock.MagicMock(), None) + mock.MagicMock(), mock.MagicMock(), None) self.client.config.dry_run = False self.assertTrue(self.client.obtain_and_enroll_certificate(domains, "example_cert")) @@ -318,8 +333,8 @@ class ClientTest(ClientTestCommon): candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") mock_parser.verb = "certonly" mock_parser.args = ["--cert-path", candidate_cert_path, - "--chain-path", candidate_chain_path, - "--fullchain-path", candidate_fullchain_path] + "--chain-path", candidate_chain_path, + "--fullchain-path", candidate_fullchain_path] cert_path, chain_path, fullchain_path = self.client.save_certificate( cert_pem, chain_pem, candidate_cert_path, candidate_chain_path, @@ -407,6 +422,7 @@ class ClientTest(ClientTestCommon): class EnhanceConfigTest(ClientTestCommon): """Tests for certbot.client.Client.enhance_config.""" + def setUp(self): super(EnhanceConfigTest, self).setUp() From 1239d7a881f7ad88c736aac526a821658cea0f96 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 17 May 2018 20:02:27 -0700 Subject: [PATCH 60/93] check platform with correct python --- certbot-nginx/certbot_nginx/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 72fc5d4de..dfc451202 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -2,11 +2,11 @@ import pkg_resources import platform -if(platform.system() == ('FreeBSD' or 'Darwin')): +if platform.system() in ('FreeBSD', 'Darwin'): server_root_tmp = "/usr/local/etc/nginx" else: server_root_tmp = "/etc/nginx" - + CLI_DEFAULTS = dict( server_root=server_root_tmp, ctl="nginx", From 94bf97b812fac7b56541992ef6ab81a6abedec49 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 May 2018 02:26:10 -0700 Subject: [PATCH 61/93] Add remaining DNS plugins to mypy.ini and sort (#6018) --- mypy.ini | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mypy.ini b/mypy.ini index 506c253a8..d00c21ae7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -8,6 +8,15 @@ check_untyped_defs = True [mypy-certbot_apache.*] check_untyped_defs = True +[mypy-certbot_dns_cloudflare.*] +check_untyped_defs = True + +[mypy-certbot_dns_cloudxns.*] +check_untyped_defs = True + +[mypy-certbot_dns_digitalocean.*] +check_untyped_defs = True + [mypy-certbot_dns_dnsimple.*] check_untyped_defs = True @@ -29,8 +38,5 @@ check_untyped_defs = True [mypy-certbot_dns_route53.*] check_untyped_defs = True -[mypy-certbot_dns_digitalocean.*] -check_untyped_defs = True - [mypy-certbot_nginx.*] check_untyped_defs = True From 250c0d6691f8fce9568fa13df0dc256181403cfe Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 May 2018 06:05:26 -0700 Subject: [PATCH 62/93] cd before running tests (#6017) When importing a module, Python first searches the current directory. See https://docs.python.org/3/tutorial/modules.html#the-module-search-path. This means that running something like `import certbot` from the root of the Certbot repo will use the local Certbot files regardless of the version installed on the system or virtual environment. Normally this behavior is fine because the local files are what we want to test, however, during our "oldest" tests, we test against older versions of our packages to make sure we're keeping compatibility. To make sure our tests use the correct versions, this commit has our tests cd to an empty temporary directory before running tests. We also had to change the package names given to pytest to be the names used in Python to import the package rather than the name of the files locally to accommodate this. --- tools/install_and_test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 59832cbc3..819f683aa 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -12,12 +12,18 @@ else pip_install="$(dirname $0)/pip_install_editable.sh" fi +temp_cwd=$(mktemp -d) +trap "rm -rf $temp_cwd" EXIT + set -x for requirement in "$@" ; do $pip_install $requirement pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] + pkg=$(echo "$pkg" | tr - _ ) # convert package names to Python import names if [ $pkg = "." ]; then pkg="certbot" fi + cd "$temp_cwd" pytest --numprocesses auto --quiet --pyargs $pkg + cd - done From 36dfd065037678bf38b7adb95ab16ed951ebbd18 Mon Sep 17 00:00:00 2001 From: Dmitry Figol Date: Fri, 18 May 2018 09:28:17 -0400 Subject: [PATCH 63/93] Prepare certbot module for mypy check untyped defs (#6005) * Prepare certbot module for mypy check untyped defs * Fix #5952 * Bump mypy to version 0.600 and fix associated bugs * Fix pylint bugs after introducing mypy * Implement Brad's suggestions * Reenabling pylint and adding nginx mypy back --- acme/acme/challenges.py | 4 +- acme/acme/crypto_util.py | 22 ++-- acme/acme/magic_typing.py | 5 +- certbot/auth_handler.py | 24 +++-- certbot/cert_manager.py | 5 +- certbot/cli.py | 54 ++++++---- certbot/client.py | 16 +-- certbot/crypto_util.py | 128 ++++++++++++------------ certbot/error_handler.py | 16 +-- certbot/hooks.py | 17 ++-- certbot/log.py | 3 +- certbot/main.py | 8 +- certbot/plugins/common.py | 6 +- certbot/plugins/disco.py | 3 +- certbot/plugins/disco_test.py | 3 +- certbot/plugins/manual.py | 5 +- certbot/plugins/selection_test.py | 3 +- certbot/plugins/standalone.py | 29 ++++-- certbot/plugins/standalone_test.py | 19 ++-- certbot/plugins/storage.py | 3 +- certbot/plugins/webroot.py | 17 ++-- certbot/renewal.py | 17 ++-- certbot/reverter.py | 12 ++- certbot/tests/auth_handler_test.py | 4 +- certbot/tests/cli_test.py | 3 +- certbot/tests/display/completer_test.py | 3 +- certbot/tests/error_handler_test.py | 6 +- certbot/tests/hook_test.py | 21 ++-- certbot/tests/log_test.py | 13 +-- certbot/tests/main_test.py | 17 ++-- certbot/tests/reporter_test.py | 18 ++-- certbot/util.py | 14 ++- mypy.ini | 6 ++ setup.py | 2 +- tools/dev_constraints.txt | 2 +- tox.ini | 2 +- 36 files changed, 316 insertions(+), 214 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index b2a4882eb..674f2c38f 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -147,9 +147,9 @@ class KeyAuthorizationChallenge(_TokenChallenge): :param response_cls: Subclass of `KeyAuthorizationChallengeResponse` that will be used to generate `response`. - + :param str typ: type of the challenge """ - + typ = NotImplemented response_cls = NotImplemented thumbprint_hash_function = ( KeyAuthorizationChallengeResponse.thumbprint_hash_function) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index ad914ca60..d0e203417 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -12,7 +12,8 @@ import josepy as jose from acme import errors # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Callable, Text, Union +from acme.magic_typing import Callable, Union, Tuple, Optional +# pylint: enable=unused-import, no-name-in-module logger = logging.getLogger(__name__) @@ -135,14 +136,23 @@ def probe_sni(name, host, port=443, timeout=300, socket_kwargs = {'source_address': source_address} - host_protocol_agnostic = None if host == '::' or host == '0' else host + host_protocol_agnostic = host + if host == '::' or host == '0': + # https://github.com/python/typeshed/pull/2136 + # while PR is not merged, we need to ignore + host_protocol_agnostic = None try: # pylint: disable=star-args - logger.debug("Attempting to connect to %s:%d%s.", host_protocol_agnostic, port, - " from {0}:{1}".format(source_address[0], source_address[1]) if \ - socket_kwargs else "") - sock = socket.create_connection((host_protocol_agnostic, port), **socket_kwargs) + logger.debug( + "Attempting to connect to %s:%d%s.", host_protocol_agnostic, port, + " from {0}:{1}".format( + source_address[0], + source_address[1] + ) if socket_kwargs else "" + ) + socket_tuple = (host_protocol_agnostic, port) # type: Tuple[Optional[str], int] + sock = socket.create_connection(socket_tuple, **socket_kwargs) # type: ignore except socket.error as error: raise errors.Error(error) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 555088cf2..471b8dfa9 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -8,6 +8,9 @@ class TypingClass(object): try: # mypy doesn't respect modifying sys.modules - from typing import * # pylint: disable=wildcard-import, unused-wildcard-import + from typing import * # pylint: disable=wildcard-import, unused-wildcard-import + # pylint: disable=unused-import + from typing import Collection, IO # type: ignore + # pylint: enable=unused-import except ImportError: sys.modules[__name__] = TypingClass() diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index caf112c61..e7d658b25 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -8,7 +8,9 @@ import zope.component from acme import challenges from acme import messages - +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import DefaultDict, Dict, List, Set, Collection +# pylint: enable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot import error_handler @@ -117,7 +119,7 @@ class AuthHandler(object): def _solve_challenges(self, aauthzrs): """Get Responses for challenges from authenticators.""" - resp = [] + resp = [] # type: Collection[acme.challenges.ChallengeResponse] all_achalls = self._get_all_achalls(aauthzrs) try: if all_achalls: @@ -133,10 +135,9 @@ class AuthHandler(object): def _get_all_achalls(self, aauthzrs): """Return all active challenges.""" - all_achalls = [] + all_achalls = [] # type: Collection[challenges.ChallengeResponse] for aauthzr in aauthzrs: all_achalls.extend(aauthzr.achalls) - return all_achalls def _respond(self, aauthzrs, resp, best_effort): @@ -146,7 +147,8 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 - chall_update = dict() + chall_update = dict() \ + # type: Dict[int, List[achallenges.KeyAuthorizationAnnotatedChallenge]] self._send_responses(aauthzrs, resp, chall_update) # Check for updated status... @@ -198,7 +200,7 @@ class AuthHandler(object): while indices_to_check and rounds < max_rounds: # TODO: Use retry-after... time.sleep(min_sleep) - all_failed_achalls = set() + all_failed_achalls = set() # type: Set[achallenges.KeyAuthorizationAnnotatedChallenge] for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( aauthzrs, index, chall_update[index]) @@ -424,7 +426,7 @@ def _find_smart_path(challbs, preferences, combinations): # max_cost is now equal to sum(indices) + 1 - best_combo = [] + best_combo = None # Set above completing all of the available challenges best_combo_cost = max_cost @@ -479,7 +481,7 @@ def _report_no_chall_path(challbs): msg += ( " You may need to use an authenticator " "plugin that can do challenges over DNS.") - logger.fatal(msg) + logger.critical(msg) raise errors.AuthorizationError(msg) @@ -522,11 +524,11 @@ def _report_failed_challs(failed_achalls): :class:`certbot.achallenges.AnnotatedChallenge`. """ - problems = dict() + problems = collections.defaultdict(list)\ + # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] for achall in failed_achalls: if achall.error: - problems.setdefault(achall.error.typ, []).append(achall) - + problems[achall.error.typ].append(achall) reporter = zope.component.getUtility(interfaces.IReporter) for achalls in six.itervalues(problems): reporter.add_message( diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index d841c1912..d1205835a 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -7,6 +7,7 @@ import re import traceback import zope.component +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -226,7 +227,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func def find_matches(candidate_lineage, return_value, acceptable_matches): """Returns a list of matches using _search_lineages.""" acceptable_matches = [func(candidate_lineage) for func in acceptable_matches] - acceptable_matches_rv = [] + acceptable_matches_rv = [] # type: List[str] for item in acceptable_matches: if isinstance(item, list): acceptable_matches_rv += item @@ -340,7 +341,7 @@ def _report_human_readable(config, parsed_certs): def _describe_certs(config, parsed_certs, parse_failures): """Print information about the certs we know about""" - out = [] + out = [] # type: List[str] notify = out.append diff --git a/certbot/cli.py b/certbot/cli.py index b71d60055..8a1ad381a 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -12,10 +12,14 @@ import sys import configargparse import six import zope.component +import zope.interface from zope.interface import interfaces as zope_interfaces from acme import challenges +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any, Dict, Optional +# pylint: enable=unused-import, no-name-in-module import certbot @@ -33,7 +37,7 @@ import certbot.plugins.selection as plugin_selection logger = logging.getLogger(__name__) # Global, to save us from a lot of argument passing within the scope of this module -helpful_parser = None +helpful_parser = None # type: Optional[HelpfulArgumentParser] # For help strings, figure out how the user ran us. # When invoked from letsencrypt-auto, sys.argv[0] is something like: @@ -196,17 +200,17 @@ def set_by_cli(var): (CLI or config file) including if the user explicitly set it to the default. Returns False if the variable was assigned a default value. """ - detector = set_by_cli.detector - if detector is None: + detector = set_by_cli.detector # type: ignore + if detector is None and helpful_parser is not None: # Setup on first run: `detector` is a weird version of config in which # the default value of every attribute is wrangled to be boolean-false plugins = plugins_disco.PluginsRegistry.find_all() # reconstructed_args == sys.argv[1:], or whatever was passed to main() reconstructed_args = helpful_parser.args + [helpful_parser.verb] - detector = set_by_cli.detector = prepare_and_parse_args( + detector = set_by_cli.detector = prepare_and_parse_args( # type: ignore plugins, reconstructed_args, detect_defaults=True) # propagate plugin requests: eg --standalone modifies config.authenticator - detector.authenticator, detector.installer = ( + detector.authenticator, detector.installer = ( # type: ignore plugin_selection.cli_plugin_requests(detector)) if not isinstance(getattr(detector, var), _Default): @@ -220,7 +224,10 @@ def set_by_cli(var): return True return False + # static housekeeping var +# functions attributed are not supported by mypy +# https://github.com/python/mypy/issues/2087 set_by_cli.detector = None # type: ignore @@ -236,8 +243,10 @@ def has_default_value(option, value): :rtype: bool """ - return (option in helpful_parser.defaults and - helpful_parser.defaults[option] == value) + if helpful_parser is not None: + return (option in helpful_parser.defaults and + helpful_parser.defaults[option] == value) + return False def option_was_set(option, value): @@ -254,11 +263,12 @@ def option_was_set(option, value): def argparse_type(variable): - "Return our argparse type function for a config variable (default: str)" + """Return our argparse type function for a config variable (default: str)""" # pylint: disable=protected-access - for action in helpful_parser.parser._actions: - if action.type is not None and action.dest == variable: - return action.type + if helpful_parser is not None: + for action in helpful_parser.parser._actions: + if action.type is not None and action.dest == variable: + return action.type return str def read_file(filename, mode="rb"): @@ -291,10 +301,12 @@ def flag_default(name): def config_help(name, hidden=False): """Extract the help message for an `.IConfig` attribute.""" + # pylint: disable=no-member if hidden: return argparse.SUPPRESS else: - return interfaces.IConfig[name].__doc__ + field = interfaces.IConfig.__getitem__(name) # type: zope.interface.interface.Attribute + return field.__doc__ class HelpfulArgumentGroup(object): @@ -473,7 +485,7 @@ class HelpfulArgumentParser(object): HELP_TOPICS += list(self.VERBS) + self.COMMANDS_TOPICS + ["manage"] plugin_names = list(plugins) - self.help_topics = HELP_TOPICS + plugin_names + [None] + self.help_topics = HELP_TOPICS + plugin_names + [None] # type: ignore self.detect_defaults = detect_defaults self.args = args @@ -492,8 +504,11 @@ class HelpfulArgumentParser(object): short_usage = self._usage_string(plugins, self.help_arg) self.visible_topics = self.determine_help_topics(self.help_arg) - self.groups = {} # elements are added by .add_group() - self.defaults = {} # elements are added by .parse_args() + + # elements are added by .add_group() + self.groups = {} # type: Dict[str, argparse._ArgumentGroup] + # elements are added by .parse_args() + self.defaults = {} # type: Dict[str, Any] self.parser = configargparse.ArgParser( prog="certbot", @@ -805,7 +820,6 @@ class HelpfulArgumentParser(object): if self.help_arg: for v in verbs: self.groups[topic].add_argument(v, help=VERB_HELP_MAP[v]["short"]) - return HelpfulArgumentGroup(self, topic) def add_plugin_args(self, plugins): @@ -1296,14 +1310,14 @@ def _paths_parser(helpful): verb = helpful.help_arg cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked." - section = ["paths", "install", "revoke", "certonly", "manage"] + sections = ["paths", "install", "revoke", "certonly", "manage"] if verb == "certonly": - add(section, "--cert-path", type=os.path.abspath, + add(sections, "--cert-path", type=os.path.abspath, default=flag_default("auth_cert_path"), help=cph) elif verb == "revoke": - add(section, "--cert-path", type=read_file, required=True, help=cph) + add(sections, "--cert-path", type=read_file, required=True, help=cph) else: - add(section, "--cert-path", type=os.path.abspath, help=cph) + add(sections, "--cert-path", type=os.path.abspath, help=cph) section = "paths" if verb in ("install", "revoke"): diff --git a/certbot/client.py b/certbot/client.py index cdfbdc252..1932ab83e 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -5,7 +5,9 @@ import os import platform from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa +# https://github.com/python/typeshed/blob/master/third_party/ +# 2/cryptography/hazmat/primitives/asymmetric/rsa.pyi +from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key # type: ignore import josepy as jose import OpenSSL import zope.component @@ -160,11 +162,11 @@ def register(config, account_storage, tos_cb=None): config.email = None # Each new registration shall use a fresh new key - key = jose.JWKRSA(key=jose.ComparableRSAKey( - rsa.generate_private_key( + rsa_key = generate_private_key( public_exponent=65537, key_size=config.rsa_key_size, - backend=default_backend()))) + backend=default_backend()) + key = jose.JWKRSA(key=jose.ComparableRSAKey(rsa_key)) acme = acme_from_config_key(config, key) # TODO: add phone? regr = perform_registration(acme, config, tos_cb) @@ -609,8 +611,10 @@ def validate_key_csr(privkey, csr=None): if csr.form == "der": csr_obj = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_ASN1, csr.data) - csr = util.CSR(csr.file, OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, csr_obj), "pem") + cert_buffer = OpenSSL.crypto.dump_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, csr_obj + ) + csr = util.CSR(csr.file, cert_buffer, "pem") # If CSR is provided, it must be readable and valid. if csr.data and not crypto_util.valid_csr(csr.data): diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 756bd7565..b5ad16db1 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -8,15 +8,18 @@ import hashlib import logging import os -import OpenSSL + import pyrfc3339 import six import zope.component +from OpenSSL import crypto +from OpenSSL import SSL # type: ignore from cryptography.hazmat.backends import default_backend -from cryptography import x509 # type: ignore +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util - +from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot import util @@ -47,7 +50,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): try: key_pem = make_key(key_size) except ValueError as err: - logger.exception(err) + logger.error("", exc_info=True) raise err config = zope.component.getUtility(interfaces.IConfig) @@ -111,11 +114,11 @@ def valid_csr(csr): """ try: - req = OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, csr) + req = crypto.load_certificate_request( + crypto.FILETYPE_PEM, csr) return req.verify(req.get_pubkey()) - except OpenSSL.crypto.Error as error: - logger.debug(error, exc_info=True) + except crypto.Error: + logger.debug("", exc_info=True) return False @@ -129,13 +132,13 @@ def csr_matches_pubkey(csr, privkey): :rtype: bool """ - req = OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, csr) - pkey = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, privkey) + req = crypto.load_certificate_request( + crypto.FILETYPE_PEM, csr) + pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, privkey) try: return req.verify(pkey) - except OpenSSL.crypto.Error as error: - logger.debug(error, exc_info=True) + except crypto.Error: + logger.debug("", exc_info=True) return False @@ -145,26 +148,26 @@ def import_csr_file(csrfile, data): :param str csrfile: CSR filename :param str data: contents of the CSR file - :returns: (`OpenSSL.crypto.FILETYPE_PEM`, + :returns: (`crypto.FILETYPE_PEM`, util.CSR object representing the CSR, list of domains requested in the CSR) :rtype: tuple """ - PEM = OpenSSL.crypto.FILETYPE_PEM - load = OpenSSL.crypto.load_certificate_request + PEM = crypto.FILETYPE_PEM + load = crypto.load_certificate_request try: # Try to parse as DER first, then fall back to PEM. - csr = load(OpenSSL.crypto.FILETYPE_ASN1, data) - except OpenSSL.crypto.Error: + csr = load(crypto.FILETYPE_ASN1, data) + except crypto.Error: try: csr = load(PEM, data) - except OpenSSL.crypto.Error: + except crypto.Error: raise errors.Error("Failed to parse CSR file: {0}".format(csrfile)) domains = _get_names_from_loaded_cert_or_req(csr) # Internally we always use PEM, so re-encode as PEM before returning. - data_pem = OpenSSL.crypto.dump_certificate_request(PEM, csr) + data_pem = crypto.dump_certificate_request(PEM, csr) return PEM, util.CSR(file=csrfile, data=data_pem, form="pem"), domains @@ -178,9 +181,9 @@ def make_key(bits): """ assert bits >= 1024 # XXX - key = OpenSSL.crypto.PKey() - key.generate_key(OpenSSL.crypto.TYPE_RSA, bits) - return OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key) + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) + return crypto.dump_privatekey(crypto.FILETYPE_PEM, key) def valid_privkey(privkey): @@ -193,9 +196,9 @@ def valid_privkey(privkey): """ try: - return OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, privkey).check() - except (TypeError, OpenSSL.crypto.Error): + return crypto.load_privatekey( + crypto.FILETYPE_PEM, privkey).check() + except (TypeError, crypto.Error): return False @@ -224,13 +227,14 @@ def verify_renewable_cert_sig(renewable_cert): :raises errors.Error: If signature verification fails. """ try: - with open(renewable_cert.chain, 'rb') as chain: - chain, _ = pyopenssl_load_certificate(chain.read()) - with open(renewable_cert.cert, 'rb') as cert: - cert = x509.load_pem_x509_certificate(cert.read(), default_backend()) + with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] + chain, _ = pyopenssl_load_certificate(chain_file.read()) + with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] + cert = x509.load_pem_x509_certificate( + cert_file.read(), default_backend()) hash_name = cert.signature_hash_algorithm.name - OpenSSL.crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) - except (IOError, ValueError, OpenSSL.crypto.Error) as e: + crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) + except (IOError, ValueError, crypto.Error) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) @@ -246,11 +250,11 @@ def verify_cert_matches_priv_key(cert_path, key_path): :raises errors.Error: If they don't match. """ try: - context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) + context = SSL.Context(SSL.SSLv23_METHOD) context.use_certificate_file(cert_path) context.use_privatekey_file(key_path) context.check_privatekey() - except (IOError, OpenSSL.SSL.Error) as e: + except (IOError, SSL.Error) as e: error_str = "verifying the cert located at {0} matches the \ private key located at {1} has failed. \ Details: {2}".format(cert_path, @@ -267,12 +271,12 @@ def verify_fullchain(renewable_cert): :raises errors.Error: If cert and chain do not combine to fullchain. """ try: - with open(renewable_cert.chain) as chain: - chain = chain.read() - with open(renewable_cert.cert) as cert: - cert = cert.read() - with open(renewable_cert.fullchain) as fullchain: - fullchain = fullchain.read() + with open(renewable_cert.chain) as chain_file: # type: IO[str] + chain = chain_file.read() + with open(renewable_cert.cert) as cert_file: # type: IO[str] + cert = cert_file.read() + with open(renewable_cert.fullchain) as fullchain_file: # type: IO[str] + fullchain = fullchain_file.read() if (cert + chain) != fullchain: error_str = "fullchain does not match cert + chain for {0}!" error_str = error_str.format(renewable_cert.lineagename) @@ -294,43 +298,43 @@ def pyopenssl_load_certificate(data): openssl_errors = [] - for file_type in (OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1): + for file_type in (crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1): try: - return OpenSSL.crypto.load_certificate(file_type, data), file_type - except OpenSSL.crypto.Error as error: # TODO: other errors? + return crypto.load_certificate(file_type, data), file_type + except crypto.Error as error: # TODO: other errors? openssl_errors.append(error) raise errors.Error("Unable to load: {0}".format(",".join( str(error) for error in openssl_errors))) def _load_cert_or_req(cert_or_req_str, load_func, - typ=OpenSSL.crypto.FILETYPE_PEM): + typ=crypto.FILETYPE_PEM): try: return load_func(typ, cert_or_req_str) - except OpenSSL.crypto.Error as error: - logger.exception(error) + except crypto.Error: + logger.error("", exc_info=True) raise def _get_sans_from_cert_or_req(cert_or_req_str, load_func, - typ=OpenSSL.crypto.FILETYPE_PEM): + typ=crypto.FILETYPE_PEM): # pylint: disable=protected-access return acme_crypto_util._pyopenssl_cert_or_req_san(_load_cert_or_req( cert_or_req_str, load_func, typ)) -def get_sans_from_cert(cert, typ=OpenSSL.crypto.FILETYPE_PEM): +def get_sans_from_cert(cert, typ=crypto.FILETYPE_PEM): """Get a list of Subject Alternative Names from a certificate. :param str cert: Certificate (encoded). - :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` + :param typ: `crypto.FILETYPE_PEM` or `crypto.FILETYPE_ASN1` :returns: A list of Subject Alternative Names. :rtype: list """ return _get_sans_from_cert_or_req( - cert, OpenSSL.crypto.load_certificate, typ) + cert, crypto.load_certificate, typ) def _get_names_from_cert_or_req(cert_or_req, load_func, typ): @@ -343,24 +347,24 @@ def _get_names_from_loaded_cert_or_req(loaded_cert_or_req): return acme_crypto_util._pyopenssl_cert_or_req_all_names(loaded_cert_or_req) -def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): +def get_names_from_cert(csr, typ=crypto.FILETYPE_PEM): """Get a list of domains from a cert, including the CN if it is set. :param str cert: Certificate (encoded). - :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` + :param typ: `crypto.FILETYPE_PEM` or `crypto.FILETYPE_ASN1` :returns: A list of domain names. :rtype: list """ return _get_names_from_cert_or_req( - csr, OpenSSL.crypto.load_certificate, typ) + csr, crypto.load_certificate, typ) -def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): +def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. - :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in + :param list chain: List of `crypto.X509` (or wrapped in :class:`josepy.util.ComparableX509`). """ @@ -378,7 +382,7 @@ def notBefore(cert_path): :rtype: :class:`datetime.datetime` """ - return _notAfterBefore(cert_path, OpenSSL.crypto.X509.get_notBefore) + return _notAfterBefore(cert_path, crypto.X509.get_notBefore) def notAfter(cert_path): @@ -390,15 +394,15 @@ def notAfter(cert_path): :rtype: :class:`datetime.datetime` """ - return _notAfterBefore(cert_path, OpenSSL.crypto.X509.get_notAfter) + return _notAfterBefore(cert_path, crypto.X509.get_notAfter) def _notAfterBefore(cert_path, method): """Internal helper function for finding notbefore/notafter. :param str cert_path: path to a cert in PEM format - :param function method: one of ``OpenSSL.crypto.X509.get_notBefore`` - or ``OpenSSL.crypto.X509.get_notAfter`` + :param function method: one of ``crypto.X509.get_notBefore`` + or ``crypto.X509.get_notAfter`` :returns: the notBefore or notAfter value from the cert at cert_path :rtype: :class:`datetime.datetime` @@ -406,7 +410,7 @@ def _notAfterBefore(cert_path, method): """ # pylint: disable=redefined-outer-name with open(cert_path) as f: - x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, + x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) # pyopenssl always returns bytes timestamp = method(x509) @@ -443,7 +447,7 @@ def cert_and_chain_from_fullchain(fullchain_pem): :rtype: tuple """ - cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)).decode() + cert = crypto.dump_certificate(crypto.FILETYPE_PEM, + crypto.load_certificate(crypto.FILETYPE_PEM, fullchain_pem)).decode() chain = fullchain_pem[len(cert):].lstrip() return (cert, chain) diff --git a/certbot/error_handler.py b/certbot/error_handler.py index e2737711e..5e72f8153 100644 --- a/certbot/error_handler.py +++ b/certbot/error_handler.py @@ -5,6 +5,10 @@ import os import signal import traceback +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any, Callable, Dict, List, Union +# pylint: enable=unused-import, no-name-in-module + from certbot import errors logger = logging.getLogger(__name__) @@ -56,9 +60,9 @@ class ErrorHandler(object): def __init__(self, func=None, *args, **kwargs): self.call_on_regular_exit = False self.body_executed = False - self.funcs = [] - self.prev_handlers = {} - self.received_signals = [] + self.funcs = [] # type: List[Callable[[], Any]] + self.prev_handlers = {} # type: Dict[int, Union[int, None, Callable]] + self.received_signals = [] # type: List[int] if func is not None: self.register(func, *args, **kwargs) @@ -88,6 +92,7 @@ class ErrorHandler(object): return retval def register(self, func, *args, **kwargs): + # type: (Callable, *Any, **Any) -> None """Sets func to be run with the given arguments during cleanup. :param function func: function to be called in case of an error @@ -101,9 +106,8 @@ class ErrorHandler(object): while self.funcs: try: self.funcs[-1]() - except Exception as error: # pylint: disable=broad-except - logger.error("Encountered exception during recovery") - logger.exception(error) + except Exception: # pylint: disable=broad-except + logger.error("Encountered exception during recovery: ", exc_info=True) self.funcs.pop() def _set_signal_handlers(self): diff --git a/certbot/hooks.py b/certbot/hooks.py index b5c9046e9..d5239a437 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -6,6 +6,7 @@ import os from subprocess import Popen, PIPE +from acme.magic_typing import Set, List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import util @@ -76,7 +77,8 @@ def pre_hook(config): if cmd: _run_pre_hook_if_necessary(cmd) -pre_hook.already = set() # type: ignore + +executed_pre_hooks = set() # type: Set[str] def _run_pre_hook_if_necessary(command): @@ -88,12 +90,12 @@ def _run_pre_hook_if_necessary(command): :param str command: pre-hook to be run """ - if command in pre_hook.already: + if command in executed_pre_hooks: logger.info("Pre-hook command already run, skipping: %s", command) else: logger.info("Running pre-hook command: %s", command) _run_hook(command) - pre_hook.already.add(command) + executed_pre_hooks.add(command) def post_hook(config): @@ -127,7 +129,8 @@ def post_hook(config): logger.info("Running post-hook command: %s", cmd) _run_hook(cmd) -post_hook.eventually = [] # type: ignore + +post_hooks = [] # type: List[str] def _run_eventually(command): @@ -139,13 +142,13 @@ def _run_eventually(command): :param str command: post-hook to register to be run """ - if command not in post_hook.eventually: - post_hook.eventually.append(command) + if command not in post_hooks: + post_hooks.append(command) def run_saved_post_hooks(): """Run any post hooks that were saved up in the course of the 'renew' verb""" - for cmd in post_hook.eventually: + for cmd in post_hooks: logger.info("Running post-hook command: %s", cmd) _run_hook(cmd) diff --git a/certbot/log.py b/certbot/log.py index face93cb3..89626af99 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -191,9 +191,8 @@ class MemoryHandler(logging.handlers.MemoryHandler): only happens when flush(force=True) is called. """ - def __init__(self, target=None): + def __init__(self, target=None, capacity=10000): # capacity doesn't matter because should_flush() is overridden - capacity = float('inf') super(MemoryHandler, self).__init__(capacity, target=target) def close(self): diff --git a/certbot/main.py b/certbot/main.py index 8a9a37084..6c1d82793 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -11,6 +11,7 @@ import josepy as jose import zope.component from acme import errors as acme_errors +from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module import certbot @@ -520,8 +521,8 @@ def _determine_account(config): config, account_storage, tos_cb=_tos_cb) except errors.MissingCommandlineFlag: raise - except errors.Error as error: - logger.debug(error, exc_info=True) + except errors.Error: + logger.debug("", exc_info=True) raise errors.Error( "Unable to register an account with ACME server") @@ -1271,7 +1272,8 @@ def set_displayer(config): """ if config.quiet: config.noninteractive_mode = True - displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) + displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) \ + # type: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] elif config.noninteractive_mode: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 147d9e21a..ee1af4978 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -11,6 +11,8 @@ import zope.interface from josepy import util as jose_util +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from certbot import achallenges # pylint: disable=unused-import from certbot import constants from certbot import crypto_util from certbot import errors @@ -331,8 +333,8 @@ class ChallengePerformer(object): def __init__(self, configurator): self.configurator = configurator - self.achalls = [] - self.indices = [] + self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] + self.indices = [] # type: List[int] def add_chall(self, achall, idx=None): """Store challenge to be performed when perform() is called. diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 062c11650..c33a56785 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -10,6 +10,7 @@ from collections import OrderedDict import zope.interface import zope.interface.verify +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from certbot import constants from certbot import errors from certbot import interfaces @@ -189,7 +190,7 @@ class PluginsRegistry(collections.Mapping): @classmethod def find_all(cls): """Find plugins using setuptools entry points.""" - plugins = {} + plugins = {} # type: Dict[str, PluginEntryPoint] # pylint: disable=not-callable entry_points = itertools.chain( pkg_resources.iter_entry_points( diff --git a/certbot/plugins/disco_test.py b/certbot/plugins/disco_test.py index 220b902b3..720b90b16 100644 --- a/certbot/plugins/disco_test.py +++ b/certbot/plugins/disco_test.py @@ -8,6 +8,7 @@ import pkg_resources import six import zope.interface +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces @@ -250,7 +251,7 @@ class PluginsRegistryTest(unittest.TestCase): self.plugin_ep.prepare.assert_called_once_with() def test_prepare_order(self): - order = [] + order = [] # type: List[str] plugins = dict( (c, mock.MagicMock(prepare=functools.partial(order.append, c))) for c in string.ascii_letters) diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 614449d34..53533d35a 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -5,7 +5,9 @@ import zope.component import zope.interface from acme import challenges +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module +from certbot import achallenges # pylint: disable=unused-import from certbot import interfaces from certbot import errors from certbot import hooks @@ -98,7 +100,8 @@ when it receives a TLS ClientHello with the SNI extension set to super(Authenticator, self).__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() - self.env = dict() + self.env = dict() \ + # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] self.tls_sni_01 = None @classmethod diff --git a/certbot/plugins/selection_test.py b/certbot/plugins/selection_test.py index 4112810a2..ab480544a 100644 --- a/certbot/plugins/selection_test.py +++ b/certbot/plugins/selection_test.py @@ -6,6 +6,7 @@ import unittest import mock import zope.component +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.display import util as display_util from certbot.tests import util as test_util from certbot import interfaces @@ -47,7 +48,7 @@ class PickPluginTest(unittest.TestCase): self.default = None self.reg = mock.MagicMock() self.question = "Question?" - self.ifaces = [] + self.ifaces = [] # type: List[interfaces.IPlugin] def _call(self): from certbot.plugins.selection import pick_plugin diff --git a/certbot/plugins/standalone.py b/certbot/plugins/standalone.py index 817403bd3..cb2e69511 100644 --- a/certbot/plugins/standalone.py +++ b/certbot/plugins/standalone.py @@ -3,6 +3,8 @@ import argparse import collections import logging import socket +# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi +from socket import errno as socket_errors # type: ignore import OpenSSL import six @@ -10,7 +12,10 @@ import zope.interface from acme import challenges from acme import standalone as acme_standalone +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import DefaultDict, Dict, Set, Tuple, List, Type, TYPE_CHECKING +from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces @@ -18,6 +23,11 @@ from certbot.plugins import common logger = logging.getLogger(__name__) +if TYPE_CHECKING: + ServedType = DefaultDict[ + acme_standalone.BaseDualNetworkedServers, + Set[achallenges.KeyAuthorizationAnnotatedChallenge] + ] class ServerManager(object): """Standalone servers manager. @@ -33,7 +43,7 @@ class ServerManager(object): """ def __init__(self, certs, http_01_resources): - self._instances = {} + self._instances = {} # type: Dict[int, acme_standalone.BaseDualNetworkedServers] self.certs = certs self.http_01_resources = http_01_resources @@ -59,7 +69,8 @@ class ServerManager(object): address = (listenaddr, port) try: if challenge_type is challenges.TLSSNI01: - servers = acme_standalone.TLSSNI01DualNetworkedServers(address, self.certs) + servers = acme_standalone.TLSSNI01DualNetworkedServers( + address, self.certs) # type: acme_standalone.BaseDualNetworkedServers else: # challenges.HTTP01 servers = acme_standalone.HTTP01DualNetworkedServers( address, self.http_01_resources) @@ -103,7 +114,8 @@ class ServerManager(object): return self._instances.copy() -SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01] +SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01] \ +# type: List[Type[challenges.KeyAuthorizationChallenge]] class SupportedChallengesAction(argparse.Action): @@ -179,14 +191,15 @@ class Authenticator(common.Plugin): self.key = OpenSSL.crypto.PKey() self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) - self.served = collections.defaultdict(set) + self.served = collections.defaultdict(set) # type: ServedType # Stuff below is shared across threads (i.e. servers read # values, main thread writes). Due to the nature of CPython's # GIL, the operations are safe, c.f. # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe - self.certs = {} - self.http_01_resources = set() + self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] + self.http_01_resources = set() \ + # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] self.servers = ServerManager(self.certs, self.http_01_resources) @@ -265,13 +278,13 @@ class Authenticator(common.Plugin): def _handle_perform_error(error): - if error.socket_error.errno == socket.errno.EACCES: + if error.socket_error.errno == socket_errors.EACCES: raise errors.PluginError( "Could not bind TCP port {0} because you don't have " "the appropriate permissions (for example, you " "aren't running this program as " "root).".format(error.port)) - elif error.socket_error.errno == socket.errno.EADDRINUSE: + elif error.socket_error.errno == socket_errors.EADDRINUSE: display = zope.component.getUtility(interfaces.IDisplay) msg = ( "Could not bind TCP port {0} because it is already in " diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 5227bc59e..47f44ff77 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -2,12 +2,18 @@ import argparse import socket import unittest +# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi +from socket import errno as socket_errors # type: ignore import josepy as jose import mock import six +import OpenSSL.crypto # pylint: disable=unused-import + from acme import challenges +from acme import standalone as acme_standalone # pylint: disable=unused-import +from acme.magic_typing import Dict, Tuple, Set # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors @@ -21,8 +27,9 @@ class ServerManagerTest(unittest.TestCase): def setUp(self): from certbot.plugins.standalone import ServerManager - self.certs = {} - self.http_01_resources = {} + self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] + self.http_01_resources = {} \ + # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] self.mgr = ServerManager(self.certs, self.http_01_resources) def test_init(self): @@ -159,7 +166,7 @@ class AuthenticatorTest(unittest.TestCase): @test_util.patch_get_utility() def test_perform_eaddrinuse_retry(self, mock_get_utility): mock_utility = mock_get_utility() - errno = socket.errno.EADDRINUSE + errno = socket_errors.EADDRINUSE error = errors.StandaloneBindError(mock.MagicMock(errno=errno), -1) self.auth.servers.run.side_effect = [error] + 2 * [mock.MagicMock()] mock_yesno = mock_utility.yesno @@ -174,7 +181,7 @@ class AuthenticatorTest(unittest.TestCase): mock_yesno = mock_utility.yesno mock_yesno.return_value = False - errno = socket.errno.EADDRINUSE + errno = socket_errors.EADDRINUSE self.assertRaises(errors.PluginError, self._fail_perform, errno) self._assert_correct_yesno_call(mock_yesno) @@ -184,11 +191,11 @@ class AuthenticatorTest(unittest.TestCase): self.assertFalse(yesno_kwargs.get("default", True)) def test_perform_eacces(self): - errno = socket.errno.EACCES + errno = socket_errors.EACCES self.assertRaises(errors.PluginError, self._fail_perform, errno) def test_perform_unexpected_socket_error(self): - errno = socket.errno.ENOTCONN + errno = socket_errors.ENOTCONN self.assertRaises( errors.StandaloneBindError, self._fail_perform, errno) diff --git a/certbot/plugins/storage.py b/certbot/plugins/storage.py index a0c3f8564..9472a1ebb 100644 --- a/certbot/plugins/storage.py +++ b/certbot/plugins/storage.py @@ -3,6 +3,7 @@ import json import logging import os +from acme.magic_typing import Any, Dict # pylint: disable=unused-import, no-name-in-module from certbot import errors logger = logging.getLogger(__name__) @@ -38,7 +39,7 @@ class PluginStorage(object): :raises .errors.PluginStorageError: when unable to open or read the file """ - data = dict() + data = dict() # type: Dict[str, Any] filedata = "" try: with open(self._storagepath, 'r') as fh: diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 6328b16ef..5d0d7d586 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -10,8 +10,12 @@ import six import zope.component import zope.interface -from acme import challenges +from acme import challenges # pylint: disable=unused-import +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict, Set, DefaultDict, List +# pylint: enable=unused-import, no-name-in-module +from certbot import achallenges # pylint: disable=unused-import from certbot import cli from certbot import errors from certbot import interfaces @@ -64,10 +68,11 @@ to serve all files under specified web root ({0}).""" def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.full_roots = {} - self.performed = collections.defaultdict(set) + self.full_roots = {} # type: Dict[str, str] + self.performed = collections.defaultdict(set) \ + # type: DefaultDict[str, Set[achallenges.KeyAuthorizationAnnotatedChallenge]] # stack of dirs successfully created by this authenticator - self._created_dirs = [] + self._created_dirs = [] # type: List[str] def prepare(self): # pylint: disable=missing-docstring pass @@ -156,7 +161,6 @@ to serve all files under specified web root ({0}).""" " --help webroot for examples.") for name, path in path_map.items(): self.full_roots[name] = os.path.join(path, challenges.HTTP01.URI_ROOT_PATH) - logger.debug("Creating root challenges validation dir at %s", self.full_roots[name]) @@ -207,7 +211,6 @@ to serve all files under specified web root ({0}).""" os.umask(old_umask) self.performed[root_path].add(achall) - return response def cleanup(self, achalls): # pylint: disable=missing-docstring @@ -219,7 +222,7 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - not_removed = [] + not_removed = [] # type: List[str] while len(self._created_dirs) > 0: path = self._created_dirs.pop() try: diff --git a/certbot/renewal.py b/certbot/renewal.py index 4651eeb36..0a6568426 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -11,6 +11,8 @@ import zope.component import OpenSSL +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module + from certbot import cli from certbot import crypto_util from certbot import errors @@ -59,8 +61,8 @@ def _reconstitute(config, full_path): """ try: renewal_candidate = storage.RenewableCert(full_path, config) - except (errors.CertStorageError, IOError) as exc: - logger.warning(exc) + except (errors.CertStorageError, IOError): + logger.warning("", exc_info=True) logger.warning("Renewal configuration file %s is broken. Skipping.", full_path) logger.debug("Traceback was:\n%s", traceback.format_exc()) return None @@ -133,14 +135,15 @@ def _restore_plugin_configs(config, renewalparams): # longer defined, stored copies of that parameter will be # deserialized as strings by this logic even if they were # originally meant to be some other type. + plugin_prefixes = [] # type: List[str] if renewalparams["authenticator"] == "webroot": _restore_webroot_config(config, renewalparams) - plugin_prefixes = [] else: - plugin_prefixes = [renewalparams["authenticator"]] + plugin_prefixes.append(renewalparams["authenticator"]) - if renewalparams.get("installer", None) is not None: + if renewalparams.get("installer") is not None: plugin_prefixes.append(renewalparams["installer"]) + for plugin_prefix in set(plugin_prefixes): plugin_prefix = plugin_prefix.replace('-', '_') for config_item, config_value in six.iteritems(renewalparams): @@ -316,13 +319,13 @@ def report(msgs, category): def _renew_describe_results(config, renew_successes, renew_failures, renew_skipped, parse_failures): - out = [] + out = [] # type: List[str] notify = out.append disp = zope.component.getUtility(interfaces.IDisplay) def notify_error(err): """Notify and log errors.""" - notify(err) + notify(str(err)) logger.error(err) if config.dry_run: diff --git a/certbot/reverter.py b/certbot/reverter.py index 15ad1a987..683c0cc32 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -82,8 +82,10 @@ class Reverter(object): self._recover_checkpoint(self.config.temp_checkpoint_dir) except errors.ReverterError: # We have a partial or incomplete recovery - logger.fatal("Incomplete or failed recovery for %s", - self.config.temp_checkpoint_dir) + logger.critical( + "Incomplete or failed recovery for %s", + self.config.temp_checkpoint_dir, + ) raise errors.ReverterError("Unable to revert temporary config") def rollback_checkpoints(self, rollback=1): @@ -123,7 +125,7 @@ class Reverter(object): try: self._recover_checkpoint(cp_dir) except errors.ReverterError: - logger.fatal("Failed to load checkpoint during rollback") + logger.critical("Failed to load checkpoint during rollback") raise errors.ReverterError( "Unable to load checkpoint during rollback") rollback -= 1 @@ -457,7 +459,7 @@ class Reverter(object): self._recover_checkpoint(self.config.in_progress_dir) except errors.ReverterError: # We have a partial or incomplete recovery - logger.fatal("Incomplete or failed recovery for IN_PROGRESS " + logger.critical("Incomplete or failed recovery for IN_PROGRESS " "checkpoint - %s", self.config.in_progress_dir) raise errors.ReverterError( @@ -494,7 +496,7 @@ class Reverter(object): "Certbot probably shut down unexpectedly", os.linesep, path) except (IOError, OSError): - logger.fatal( + logger.critical( "Unable to remove filepaths contained within %s", file_list) raise errors.ReverterError( "Unable to remove filepaths contained within " diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 9a8a13498..76d1df90f 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -10,6 +10,7 @@ import zope.component from acme import challenges from acme import client as acme_client from acme import messages +from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors @@ -354,12 +355,13 @@ class PollChallengesTest(unittest.TestCase): acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []) ] - self.chall_update = {} + self.chall_update = {} # type: Dict[int, achallenges.KeyAuthorizationAnnotatedChallenge] for i, aauthzr in enumerate(self.aauthzrs): self.chall_update[i] = [ challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) for challb in aauthzr.authzr.body.challenges] + @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 1bba6991a..979cd97c1 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -495,7 +495,8 @@ class SetByCliTest(unittest.TestCase): for v in ('manual', 'manual_auth_hook', 'manual_public_ip_logging_ok'): self.assertTrue(_call_set_by_cli(v, args, verb)) - cli.set_by_cli.detector = None + # https://github.com/python/mypy/issues/2087 + cli.set_by_cli.detector = None # type: ignore args = ['--manual-auth-hook', 'command'] for v in ('manual_auth_hook', 'manual_public_ip_logging_ok'): diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 333acf2b3..ac01103b8 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -8,6 +8,7 @@ import unittest import mock from six.moves import reload_module # pylint: disable=import-error +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.tests.util import TempDirTestCase class CompleterTest(TempDirTestCase): @@ -21,7 +22,7 @@ class CompleterTest(TempDirTestCase): if self.tempdir[-1] != os.sep: self.tempdir += os.sep - self.paths = [] + self.paths = [] # type: List[str] # create some files and directories in temp_dir for c in string.ascii_lowercase: path = os.path.join(self.tempdir, c) diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index d4c48c242..a4a65e2d4 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -6,6 +6,9 @@ import sys import unittest import mock +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Callable, Dict, Union +# pylint: enable=unused-import, no-name-in-module def get_signals(signums): @@ -23,8 +26,7 @@ def set_signals(sig_handler_dict): def signal_receiver(signums): """Context manager to catch signals""" signals = [] - prev_handlers = {} - prev_handlers = get_signals(signums) + prev_handlers = get_signals(signums) # type: Dict[int, Union[int, None, Callable]] set_signals(dict((s, lambda s, _: signals.append(s)) for s in signums)) yield signals set_signals(prev_handlers) diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 8619a1a2e..c9cfc69f9 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -5,6 +5,7 @@ import unittest import mock +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot.tests import util @@ -106,8 +107,8 @@ class PreHookTest(HookTest): super(PreHookTest, self).tearDown() def _reset_pre_hook_already(self): - from certbot.hooks import pre_hook - pre_hook.already.clear() + from certbot.hooks import executed_pre_hooks + executed_pre_hooks.clear() def test_certonly(self): self.config.verb = "certonly" @@ -184,8 +185,8 @@ class PostHookTest(HookTest): super(PostHookTest, self).tearDown() def _reset_post_hook_eventually(self): - from certbot.hooks import post_hook - post_hook.eventually = [] + from certbot.hooks import post_hooks + del post_hooks[:] def test_certonly_and_run_with_hook(self): for verb in ("certonly", "run",): @@ -238,8 +239,8 @@ class PostHookTest(HookTest): self.assertEqual(self._get_eventually(), expected) def _get_eventually(self): - from certbot.hooks import post_hook - return post_hook.eventually + from certbot.hooks import post_hooks + return post_hooks class RunSavedPostHooksTest(HookTest): @@ -248,23 +249,23 @@ class RunSavedPostHooksTest(HookTest): @classmethod def _call(cls, *args, **kwargs): from certbot.hooks import run_saved_post_hooks - return run_saved_post_hooks(*args, **kwargs) + return run_saved_post_hooks() def _call_with_mock_execute_and_eventually(self, *args, **kwargs): """Call run_saved_post_hooks but mock out execute and eventually - certbot.hooks.post_hook.eventually is replaced with + certbot.hooks.post_hooks is replaced with self.eventually. The mock execute object is returned rather than the return value of run_saved_post_hooks. """ - eventually_path = "certbot.hooks.post_hook.eventually" + eventually_path = "certbot.hooks.post_hooks" with mock.patch(eventually_path, new=self.eventually): return self._call_with_mock_execute(*args, **kwargs) def setUp(self): super(RunSavedPostHooksTest, self).setUp() - self.eventually = [] + self.eventually = [] # type: List[str] def test_empty(self): self.assertFalse(self._call_with_mock_execute_and_eventually().called) diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 549d2c5e1..c5991347e 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -10,6 +10,7 @@ import mock import six from acme import messages +from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module from certbot import constants from certbot import errors @@ -21,9 +22,9 @@ class PreArgParseSetupTest(unittest.TestCase): """Tests for certbot.log.pre_arg_parse_setup.""" @classmethod - def _call(cls, *args, **kwargs): + def _call(cls, *args, **kwargs): # pylint: disable=unused-argument from certbot.log import pre_arg_parse_setup - return pre_arg_parse_setup(*args, **kwargs) + return pre_arg_parse_setup() @mock.patch('certbot.log.sys') @mock.patch('certbot.log.pre_arg_parse_except_hook') @@ -38,16 +39,16 @@ class PreArgParseSetupTest(unittest.TestCase): mock_root_logger.setLevel.assert_called_once_with(logging.DEBUG) self.assertEqual(mock_root_logger.addHandler.call_count, 2) - MemoryHandler = logging.handlers.MemoryHandler - memory_handler = None + memory_handler = None # type: Optional[logging.handlers.MemoryHandler] for call in mock_root_logger.addHandler.call_args_list: handler = call[0][0] - if memory_handler is None and isinstance(handler, MemoryHandler): + if memory_handler is None and isinstance(handler, logging.handlers.MemoryHandler): memory_handler = handler + target = memory_handler.target # type: ignore else: self.assertTrue(isinstance(handler, logging.StreamHandler)) self.assertTrue( - isinstance(memory_handler.target, logging.StreamHandler)) + isinstance(target, logging.StreamHandler)) mock_register.assert_called_once_with(logging.shutdown) mock_sys.excepthook(1, 2, 3) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 0986ff060..14cde27ee 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -16,12 +16,14 @@ import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import account from certbot import cli from certbot import constants from certbot import configuration from certbot import crypto_util from certbot import errors +from certbot import interfaces # pylint: disable=unused-import from certbot import main from certbot import updater from certbot import util @@ -600,14 +602,14 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met if mockisfile: orig_open = os.path.isfile - def mock_isfile(fn, *args, **kwargs): + def mock_isfile(fn, *args, **kwargs): # pylint: disable=unused-argument """Mock os.path.isfile()""" if (fn.endswith("cert") or fn.endswith("chain") or fn.endswith("privkey")): return True else: - return orig_open(fn, *args, **kwargs) + return orig_open(fn) with mock.patch("os.path.isfile") as mock_if: mock_if.side_effect = mock_isfile @@ -836,7 +838,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() stdout = six.StringIO() @@ -851,7 +853,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args_unprivileged(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() def throw_error(directory, mode, uid, strict): @@ -873,7 +875,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_init(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() stdout = six.StringIO() @@ -891,7 +893,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plugins_disco') @mock.patch('certbot.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_prepare(self, _det, mock_disco): - ifaces = [] + ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() stdout = six.StringIO() @@ -1040,9 +1042,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mock_client.obtain_certificate.return_value = (mock_certr, 'chain', mock_key, 'csr') - def write_msg(message, *args, **kwargs): + def write_msg(message, *args, **kwargs): # pylint: disable=unused-argument """Write message to stdout.""" - _, _ = args, kwargs stdout.write(message) try: diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 9ec8dca28..22e11e672 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -12,7 +12,7 @@ class ReporterTest(unittest.TestCase): from certbot import reporter self.reporter = reporter.Reporter(mock.MagicMock(quiet=False)) - self.old_stdout = sys.stdout + self.old_stdout = sys.stdout # type: ignore sys.stdout = six.StringIO() def tearDown(self): @@ -21,32 +21,32 @@ class ReporterTest(unittest.TestCase): def test_multiline_message(self): self.reporter.add_message("Line 1\nLine 2", self.reporter.LOW_PRIORITY) self.reporter.print_messages() - output = sys.stdout.getvalue() + output = sys.stdout.getvalue() # type: ignore self.assertTrue("Line 1\n" in output) self.assertTrue("Line 2" in output) def test_tty_print_empty(self): - sys.stdout.isatty = lambda: True + sys.stdout.isatty = lambda: True # type: ignore self.test_no_tty_print_empty() def test_no_tty_print_empty(self): self.reporter.print_messages() - self.assertEqual(sys.stdout.getvalue(), "") + self.assertEqual(sys.stdout.getvalue(), "") # type: ignore try: raise ValueError except ValueError: self.reporter.print_messages() - self.assertEqual(sys.stdout.getvalue(), "") + self.assertEqual(sys.stdout.getvalue(), "") # type: ignore def test_tty_successful_exit(self): - sys.stdout.isatty = lambda: True + sys.stdout.isatty = lambda: True # type: ignore self._successful_exit_common() def test_no_tty_successful_exit(self): self._successful_exit_common() def test_tty_unsuccessful_exit(self): - sys.stdout.isatty = lambda: True + sys.stdout.isatty = lambda: True # type: ignore self._unsuccessful_exit_common() def test_no_tty_unsuccessful_exit(self): @@ -55,7 +55,7 @@ class ReporterTest(unittest.TestCase): def _successful_exit_common(self): self._add_messages() self.reporter.print_messages() - output = sys.stdout.getvalue() + output = sys.stdout.getvalue() # type: ignore self.assertTrue("IMPORTANT NOTES:" in output) self.assertTrue("High" in output) self.assertTrue("Med" in output) @@ -67,7 +67,7 @@ class ReporterTest(unittest.TestCase): raise ValueError except ValueError: self.reporter.print_messages() - output = sys.stdout.getvalue() + output = sys.stdout.getvalue() # type: ignore self.assertTrue("IMPORTANT NOTES:" in output) self.assertTrue("High" in output) self.assertTrue("Med" not in output) diff --git a/certbot/util.py b/certbot/util.py index 55acd624f..8e84c29ba 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -20,6 +20,7 @@ from collections import OrderedDict import configargparse +from acme.magic_typing import Tuple, Union # pylint: disable=unused-import, no-name-in-module from certbot import constants from certbot import errors from certbot import lock @@ -218,8 +219,12 @@ def safe_open(path, mode="w", chmod=None, buffering=None): """ # pylint: disable=star-args - open_args = () if chmod is None else (chmod,) - fdopen_args = () if buffering is None else (buffering,) + open_args = () # type: Union[Tuple[()], Tuple[int]] + if chmod is not None: + open_args = (chmod,) + fdopen_args = () # type: Union[Tuple[()], Tuple[int]] + if buffering is not None: + fdopen_args = (buffering,) return os.fdopen( os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args), mode, *fdopen_args) @@ -303,9 +308,8 @@ def get_filtered_names(all_names): for name in all_names: try: filtered_names.add(enforce_le_validity(name)) - except errors.ConfigurationError as error: - logger.debug('Not suggesting name "%s"', name) - logger.debug(error) + except errors.ConfigurationError: + logger.debug('Not suggesting name "%s"', name, exc_info=True) return filtered_names diff --git a/mypy.ini b/mypy.ini index d00c21ae7..f0c99e65f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,12 @@ ignore_missing_imports = True [mypy-acme.*] check_untyped_defs = True +[mypy-acme.magic_typing_test] +ignore_errors = True + +[mypy-certbot.*] +check_untyped_defs = True + [mypy-certbot_apache.*] check_untyped_defs = True diff --git a/setup.py b/setup.py index 3760fd35b..ee0470d3a 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=0.22.1', + 'acme>0.24.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index df13cdbef..d965d4470 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -30,7 +30,7 @@ josepy==1.0.1 logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 -mypy==0.580 +mypy==0.600 ndg-httpsclient==0.3.2 oauth2client==2.0.0 pathlib2==2.3.0 diff --git a/tox.ini b/tox.ini index 8c4d6c38d..2834ef9f9 100644 --- a/tox.ini +++ b/tox.ini @@ -121,8 +121,8 @@ commands = [testenv:mypy] basepython = python3 commands = - {[base]pip_install} .[dev3] {[base]install_packages} + {[base]pip_install} .[dev3] mypy {[base]source_paths} [testenv:apacheconftest] From 366c50e28ee865f697f9e32e5b86e49dbf3ec5a2 Mon Sep 17 00:00:00 2001 From: Paul Kehrer Date: Fri, 18 May 2018 09:10:41 -0700 Subject: [PATCH 64/93] switch signature verification to use pure cryptography (#6000) * switch signature verification to use pure cryptography On systems that prevent write/execute pages this prevents a segfault that is caused by pyopenssl creating a dynamic callback in the verification helper. * switch to using a verifier for older cryptography releases also add ec support, test vectors, and a test --- certbot/crypto_util.py | 36 ++++++++++++++----- certbot/tests/crypto_util_test.py | 10 ++++++ .../tests/testdata/cert-nosans_nistp256.pem | 11 ++++++ .../tests/testdata/csr-nosans_nistp256.pem | 8 +++++ certbot/tests/testdata/nistp256_key.pem | 5 +++ 5 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 certbot/tests/testdata/cert-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/csr-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/nistp256_key.pem diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index b5ad16db1..71f6c990c 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -12,11 +12,16 @@ import os import pyrfc3339 import six import zope.component +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore -from cryptography.hazmat.backends import default_backend -# https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module @@ -228,13 +233,26 @@ def verify_renewable_cert_sig(renewable_cert): """ try: with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] - chain, _ = pyopenssl_load_certificate(chain_file.read()) + chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] - cert = x509.load_pem_x509_certificate( - cert_file.read(), default_backend()) - hash_name = cert.signature_hash_algorithm.name - crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) - except (IOError, ValueError, crypto.Error) as e: + cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) + pk = chain.public_key() + if isinstance(pk, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = pk.verifier( # type: ignore + cert.signature, PKCS1v15(), cert.signature_hash_algorithm + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + elif isinstance(pk, EllipticCurvePublicKey): + verifier = pk.verifier( + cert.signature, ECDSA(cert.signature_hash_algorithm) + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") + except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 2fe0e3d30..baf14b2ef 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -21,6 +21,9 @@ CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.load_vector('cert_512.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') SS_CERT = test_util.load_vector('cert_2048.pem') +P256_KEY = test_util.load_vector('nistp256_key.pem') +P256_CERT_PATH = test_util.vector_path('cert-nosans_nistp256.pem') +P256_CERT = test_util.load_vector('cert-nosans_nistp256.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" @@ -217,6 +220,13 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): def test_cert_sig_match(self): self.assertEqual(None, self._call(self.renewable_cert)) + def test_cert_sig_match_ec(self): + renewable_cert = mock.MagicMock() + renewable_cert.cert = P256_CERT_PATH + renewable_cert.chain = P256_CERT_PATH + renewable_cert.privkey = P256_KEY + self.assertEqual(None, self._call(renewable_cert)) + def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert = test_util.vector_path('cert_512_bad.pem') self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) diff --git a/certbot/tests/testdata/cert-nosans_nistp256.pem b/certbot/tests/testdata/cert-nosans_nistp256.pem new file mode 100644 index 000000000..4ec3f24ce --- /dev/null +++ b/certbot/tests/testdata/cert-nosans_nistp256.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBoDCCAUYCCQDCnzfUZ7TQdDAKBggqhkjOPQQDAjBYMQswCQYDVQQGEwJVUzER +MA8GA1UECAwITWljaGlnYW4xEjAQBgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwD +RUZGMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0xODA1MTUxNzIyMzlaFw0xODA2 +MTQxNzIyMzlaMFgxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhNaWNoaWdhbjESMBAG +A1UEBwwJQW5uIEFyYm9yMQwwCgYDVQQKDANFRkYxFDASBgNVBAMMC2V4YW1wbGUu +Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPPl0JauSZukvAUWv4l5VNLAY +QXhuPXYQBf4dVET3s0E5q9ZCbSe+pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tzAK +BggqhkjOPQQDAgNIADBFAiEAv8S2GXmWJqZ+j3DBfm72E1YK+HkOf+TOUHsbVR+O +Z1oCIFWNt1SPdIgRp4QAyzVk2pcTF8jDNajEMLWETDtxgRvM +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/csr-nosans_nistp256.pem b/certbot/tests/testdata/csr-nosans_nistp256.pem new file mode 100644 index 000000000..2f0a671ed --- /dev/null +++ b/certbot/tests/testdata/csr-nosans_nistp256.pem @@ -0,0 +1,8 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBFDCBugIBADBYMQswCQYDVQQGEwJVUzERMA8GA1UECAwITWljaGlnYW4xEjAQ +BgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwDRUZGMRQwEgYDVQQDDAtleGFtcGxl +LmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDz5dCWrkmbpLwFFr+JeVTSw +GEF4bj12EAX+HVRE97NBOavWQm0nvqTVG5KPRfkxZLnO11Y0D7H5A24dCPZw9Leg +ADAKBggqhkjOPQQDAgNJADBGAiEAuoZHrYA5sy2DRTdLAxJTBNHKFFKbtaGt+QaJ +A62qa8sCIQCUkSgSAiNaEnJ7r5fKphdjeORHqhpl6flYkLE3lGmGdg== +-----END CERTIFICATE REQUEST----- diff --git a/certbot/tests/testdata/nistp256_key.pem b/certbot/tests/testdata/nistp256_key.pem new file mode 100644 index 000000000..4be37e49b --- /dev/null +++ b/certbot/tests/testdata/nistp256_key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIOvXH384CyNNv2lfxvjc7hg2f7ScYoLvlk/VpINLJlGBoAoGCCqGSM49 +AwEHoUQDQgAEPPl0JauSZukvAUWv4l5VNLAYQXhuPXYQBf4dVET3s0E5q9ZCbSe+ +pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tw== +-----END EC PRIVATE KEY----- From dec97fc1269636a94a9763d6a2e641d2ada3ac6f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 May 2018 17:48:30 -0700 Subject: [PATCH 65/93] Revert "Add link to pycon issues (#5959)" This reverts commit 68359086fffca8805893bf6133c53b5f75357a7f. --- docs/contributing.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 52f08efe0..ed986c562 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -72,10 +72,6 @@ found in the `virtualenv docs`_. Find issues to work on ---------------------- -.. note:: If you're sprinting on Certbot at PyCon, you can find especially good - issues to work on during the event `here - `_. - You can find the open issues in the `github issue tracker`_. Comparatively easy ones are marked `good first issue`_. If you're starting work on something, post a comment to let others know and seek feedback on your plan From c9a206ca890c3f39c5f0dffce9862cf439b9d5a1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 21 May 2018 20:23:21 -0700 Subject: [PATCH 66/93] Get mypy passing with check_untyped_defs everywhere (#6021) * unchecked_typed_defs everywhere * fix mypy for lock_test * add magic_typing * fix mypy in letshelp * fix validator errors in compat test * fix mypy for test_driver.py * fix mypy in util.py * delint --- .../certbot_compatibility_test/test_driver.py | 48 ++++++++----------- .../certbot_compatibility_test/util.py | 12 ++--- .../certbot_compatibility_test/validator.py | 5 +- letshelp-certbot/letshelp_certbot/apache.py | 5 +- .../letshelp_certbot/apache_test.py | 16 +++++-- .../letshelp_certbot/magic_typing.py | 16 +++++++ .../letshelp_certbot/magic_typing_test.py | 41 ++++++++++++++++ mypy.ini | 46 ++---------------- tests/lock_test.py | 2 +- 9 files changed, 106 insertions(+), 85 deletions(-) create mode 100644 letshelp-certbot/letshelp_certbot/magic_typing.py create mode 100644 letshelp-certbot/letshelp_certbot/magic_typing_test.py diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index 2c6c917b3..9eea95e67 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -15,6 +15,7 @@ from six.moves import xrange # pylint: disable=import-error,redefined-builtin from acme import challenges from acme import crypto_util from acme import messages +from acme.magic_typing import List, Tuple # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors as le_errors from certbot.tests import acme_util @@ -52,9 +53,8 @@ def test_authenticator(plugin, config, temp_dir): try: responses = plugin.perform(achalls) - except le_errors.Error as error: - logger.error("Performing challenges on %s caused an error:", config) - logger.exception(error) + except le_errors.Error: + logger.error("Performing challenges on %s caused an error:", config, exc_info=True) return False success = True @@ -82,9 +82,8 @@ def test_authenticator(plugin, config, temp_dir): if success: try: plugin.cleanup(achalls) - except le_errors.Error as error: - logger.error("Challenge cleanup for %s caused an error:", config) - logger.exception(error) + except le_errors.Error: + logger.error("Challenge cleanup for %s caused an error:", config, exc_info=True) success = False if _dirs_are_unequal(config, backup): @@ -147,9 +146,8 @@ def test_deploy_cert(plugin, temp_dir, domains): try: plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path, cert_path) plugin.save() # Needed by the Apache plugin - except le_errors.Error as error: - logger.error("**** Plugin failed to deploy certificate for %s:", domain) - logger.exception(error) + except le_errors.Error: + logger.error("**** Plugin failed to deploy certificate for %s:", domain, exc_info=True) return False if not _save_and_restart(plugin, "deployed"): @@ -179,7 +177,7 @@ def test_enhancements(plugin, domains): "enhancements") return False - domains_and_info = [(domain, []) for domain in domains] + domains_and_info = [(domain, []) for domain in domains] # type: List[Tuple[str, List[bool]]] for domain, info in domains_and_info: try: @@ -192,10 +190,9 @@ def test_enhancements(plugin, domains): # Don't immediately fail because a redirect may already be enabled logger.warning("*** Plugin failed to enable redirect for %s:", domain) logger.warning("%s", error) - except le_errors.Error as error: + except le_errors.Error: logger.error("*** An error occurred while enabling redirect for %s:", - domain) - logger.exception(error) + domain, exc_info=True) if not _save_and_restart(plugin, "enhanced"): return False @@ -222,9 +219,8 @@ def _save_and_restart(plugin, title=None): plugin.save(title) plugin.restart() return True - except le_errors.Error as error: - logger.error("*** Plugin failed to save and restart server:") - logger.exception(error) + except le_errors.Error: + logger.error("*** Plugin failed to save and restart server:", exc_info=True) return False @@ -232,9 +228,8 @@ def test_rollback(plugin, config, backup): """Tests the rollback checkpoints function""" try: plugin.rollback_checkpoints(1337) - except le_errors.Error as error: - logger.error("*** Plugin raised an exception during rollback:") - logger.exception(error) + except le_errors.Error: + logger.error("*** Plugin raised an exception during rollback:", exc_info=True) return False if _dirs_are_unequal(config, backup): @@ -263,21 +258,21 @@ def _dirs_are_unequal(dir1, dir2): logger.error("The following files and directories are only " "present in one directory") if dircmp.left_only: - logger.error(dircmp.left_only) + logger.error(str(dircmp.left_only)) else: - logger.error(dircmp.right_only) + logger.error(str(dircmp.right_only)) return True elif dircmp.common_funny or dircmp.funny_files: logger.error("The following files and directories could not be " "compared:") if dircmp.common_funny: - logger.error(dircmp.common_funny) + logger.error(str(dircmp.common_funny)) else: - logger.error(dircmp.funny_files) + logger.error(str(dircmp.funny_files)) return True elif dircmp.diff_files: logger.error("The following files differ:") - logger.error(dircmp.diff_files) + logger.error(str(dircmp.diff_files)) return True for subdir in dircmp.subdirs.itervalues(): @@ -354,9 +349,8 @@ def main(): success = test_authenticator(plugin, config, temp_dir) if success and args.install: success = test_installer(args, plugin, config, temp_dir) - except errors.Error as error: - logger.error("Tests on %s raised:", config) - logger.exception(error) + except errors.Error: + logger.error("Tests on %s raised:", config, exc_info=True) success = False if success: diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index 4155944bd..6051bbc2e 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -26,12 +26,12 @@ def create_le_config(parent_dir): config = copy.deepcopy(constants.CLI_DEFAULTS) le_dir = os.path.join(parent_dir, "certbot") - config["config_dir"] = os.path.join(le_dir, "config") - config["work_dir"] = os.path.join(le_dir, "work") - config["logs_dir"] = os.path.join(le_dir, "logs_dir") - os.makedirs(config["config_dir"]) - os.mkdir(config["work_dir"]) - os.mkdir(config["logs_dir"]) + os.mkdir(le_dir) + for dir_name in ("config", "logs", "work"): + full_path = os.path.join(le_dir, dir_name) + os.mkdir(full_path) + full_name = dir_name + "_dir" + config[full_name] = full_path config["domains"] = None diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index 791fe0da2..fd2f95702 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -33,7 +33,7 @@ class Validator(object): try: presented_cert = crypto_util.probe_sni(name, host, port) except acme_errors.Error as error: - logger.exception(error) + logger.exception(str(error)) return False return presented_cert.digest("sha256") == cert.digest("sha256") @@ -86,8 +86,7 @@ class Validator(object): return False try: - _, max_age_value = max_age[0] - max_age_value = int(max_age_value) + max_age_value = int(max_age[0][1]) except ValueError: logger.error("Server responded with invalid HSTS header field") return False diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index f77a6a1b0..50f3c5ef6 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -16,6 +16,8 @@ import textwrap import six +from letshelp_certbot.magic_typing import List # pylint: disable=unused-import, no-name-in-module + _DESCRIPTION = """ Let's Help is a simple script you can run to help out the Certbot project. Since Certbot will support automatically configuring HTTPS on @@ -87,7 +89,8 @@ def copy_config(server_root, temp_dir): :rtype: `tuple` of `list` of `str` """ - copied_files, copied_dirs = [], [] + copied_files = [] # type: List[str] + copied_dirs = [] # type: List[str] dir_len = len(os.path.dirname(server_root)) for config_path, config_dirs, config_files in os.walk(server_root): diff --git a/letshelp-certbot/letshelp_certbot/apache_test.py b/letshelp-certbot/letshelp_certbot/apache_test.py index e0656ae05..a1115bc06 100644 --- a/letshelp-certbot/letshelp_certbot/apache_test.py +++ b/letshelp-certbot/letshelp_certbot/apache_test.py @@ -203,13 +203,19 @@ class LetsHelpApacheTest(unittest.TestCase): tempdir_path, "config.tar.gz")) tempdir = tar.next() - self.assertTrue(tempdir.isdir()) - self.assertEqual(tempdir.name, ".") + if tempdir is None: + self.fail("Invalid tarball!") # pragma: no cover + else: + self.assertTrue(tempdir.isdir()) + self.assertEqual(tempdir.name, ".") testdir = tar.next() - self.assertTrue(testdir.isdir()) - self.assertEqual(os.path.basename(testdir.name), - testdir_basename) + if testdir is None: + self.fail("Invalid tarball!") # pragma: no cover + else: + self.assertTrue(testdir.isdir()) + self.assertEqual(os.path.basename(testdir.name), + testdir_basename) self.assertEqual(tar.next(), None) diff --git a/letshelp-certbot/letshelp_certbot/magic_typing.py b/letshelp-certbot/letshelp_certbot/magic_typing.py new file mode 100644 index 000000000..471b8dfa9 --- /dev/null +++ b/letshelp-certbot/letshelp_certbot/magic_typing.py @@ -0,0 +1,16 @@ +"""Shim class to not have to depend on typing module in prod.""" +import sys + +class TypingClass(object): + """Ignore import errors by getting anything""" + def __getattr__(self, name): + return None + +try: + # mypy doesn't respect modifying sys.modules + from typing import * # pylint: disable=wildcard-import, unused-wildcard-import + # pylint: disable=unused-import + from typing import Collection, IO # type: ignore + # pylint: enable=unused-import +except ImportError: + sys.modules[__name__] = TypingClass() diff --git a/letshelp-certbot/letshelp_certbot/magic_typing_test.py b/letshelp-certbot/letshelp_certbot/magic_typing_test.py new file mode 100644 index 000000000..200ca03b8 --- /dev/null +++ b/letshelp-certbot/letshelp_certbot/magic_typing_test.py @@ -0,0 +1,41 @@ +"""Tests for letshelp_certbot.magic_typing.""" +import sys +import unittest + +import mock + + +class MagicTypingTest(unittest.TestCase): + """Tests for letshelp_certbot.magic_typing.""" + def test_import_success(self): + try: + import typing as temp_typing + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover + typing_class_mock = mock.MagicMock() + text_mock = mock.MagicMock() + typing_class_mock.Text = text_mock + sys.modules['typing'] = typing_class_mock + if 'letshelp_certbot.magic_typing' in sys.modules: + del sys.modules['letshelp_certbot.magic_typing'] # pragma: no cover + from letshelp_certbot.magic_typing import Text # pylint: disable=no-name-in-module + self.assertEqual(Text, text_mock) + del sys.modules['letshelp_certbot.magic_typing'] + sys.modules['typing'] = temp_typing + + def test_import_failure(self): + try: + import typing as temp_typing + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover + sys.modules['typing'] = None + if 'letshelp_certbot.magic_typing' in sys.modules: + del sys.modules['letshelp_certbot.magic_typing'] # pragma: no cover + from letshelp_certbot.magic_typing import Text # pylint: disable=no-name-in-module + self.assertTrue(Text is None) + del sys.modules['letshelp_certbot.magic_typing'] + sys.modules['typing'] = temp_typing + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/mypy.ini b/mypy.ini index f0c99e65f..188ed031f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,48 +1,10 @@ [mypy] -python_version = 2.7 -ignore_missing_imports = True - -[mypy-acme.*] check_untyped_defs = True +ignore_missing_imports = True +python_version = 2.7 [mypy-acme.magic_typing_test] ignore_errors = True -[mypy-certbot.*] -check_untyped_defs = True - -[mypy-certbot_apache.*] -check_untyped_defs = True - -[mypy-certbot_dns_cloudflare.*] -check_untyped_defs = True - -[mypy-certbot_dns_cloudxns.*] -check_untyped_defs = True - -[mypy-certbot_dns_digitalocean.*] -check_untyped_defs = True - -[mypy-certbot_dns_dnsimple.*] -check_untyped_defs = True - -[mypy-certbot_dns_dnsmadeeasy.*] -check_untyped_defs = True - -[mypy-certbot_dns_google.*] -check_untyped_defs = True - -[mypy-certbot_dns_luadns.*] -check_untyped_defs = True - -[mypy-certbot_dns_nsone.*] -check_untyped_defs = True - -[mypy-certbot_dns_rfc2136.*] -check_untyped_defs = True - -[mypy-certbot_dns_route53.*] -check_untyped_defs = True - -[mypy-certbot_nginx.*] -check_untyped_defs = True +[mypy-letshelp_certbot.magic_typing_test] +ignore_errors = True diff --git a/tests/lock_test.py b/tests/lock_test.py index 4bb2865b4..b01cc5d58 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -198,7 +198,7 @@ def report_failure(err_msg, out, err): :param str err: stderr output """ - logger.fatal(err_msg) + logger.critical(err_msg) log_output(logging.INFO, out, err) sys.exit(err_msg) From cfd4b8f3634df21370f0e21876400e46d8b5b8eb Mon Sep 17 00:00:00 2001 From: Quang Vu Date: Tue, 22 May 2018 15:32:44 -0700 Subject: [PATCH 67/93] #4242 Support multi emails register (#5994) This change will allow registering/updating account with multi emails. Detail is enclosed in #4242 * support multi emails register * add more test cases * update test to unregister before register * update create path to support multi emaill * refactor payload updating * fix typo * move command line doc to another place * revert the change for updating account registration info, added unit test * rearrange text for consistency --- acme/acme/messages.py | 2 +- certbot/cli.py | 2 +- certbot/client.py | 3 ++- certbot/interfaces.py | 4 +++- certbot/main.py | 33 +++++++++++++++++---------------- certbot/tests/main_test.py | 10 +++++++--- tests/boulder-integration.sh | 9 ++++++++- 7 files changed, 39 insertions(+), 24 deletions(-) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 03dbc3255..827a4dd11 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -285,7 +285,7 @@ class Registration(ResourceBody): if phone is not None: details.append(cls.phone_prefix + phone) if email is not None: - details.append(cls.email_prefix + email) + details.extend([cls.email_prefix + mail for mail in email.split(',')]) kwargs['contact'] = tuple(details) return cls(**kwargs) diff --git a/certbot/cli.py b/certbot/cli.py index 8a1ad381a..25319bbd8 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -430,7 +430,7 @@ VERB_HELP = [ }), ("enhance", { "short": "Add security enhancements to your existing configuration", - "opts": ("Helps to harden the TLS configration by adding security enhancements " + "opts": ("Helps to harden the TLS configuration by adding security enhancements " "to already existing configuration."), "usage": "\n\n certbot enhance [options]\n\n" }), diff --git a/certbot/client.py b/certbot/client.py index 1932ab83e..dadc3a0f8 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -185,8 +185,9 @@ def perform_registration(acme, config, tos_cb): Actually register new account, trying repeatedly if there are email problems - :param .IConfig config: Client configuration. :param acme.client.Client client: ACME client object. + :param .IConfig config: Client configuration. + :param Callable tos_cb: a callback to handle Term of Service agreement. :returns: Registration Resource. :rtype: `acme.messages.RegistrationResource` diff --git a/certbot/interfaces.py b/certbot/interfaces.py index c96f6bd51..6233e3592 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -201,7 +201,9 @@ class IConfig(zope.interface.Interface): """ server = zope.interface.Attribute("ACME Directory Resource URI.") email = zope.interface.Attribute( - "Email used for registration and recovery contact. (default: Ask)") + "Email used for registration and recovery contact. Use comma to " + "register multiple emails, ex: u1@example.com,u2@example.com. " + "(default: Ask).") rsa_key_size = zope.interface.Attribute("Size of the RSA key.") must_staple = zope.interface.Attribute( "Adds the OCSP Must Staple extension to the certificate. " diff --git a/certbot/main.py b/certbot/main.py index 6c1d82793..dad1b793e 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -487,6 +487,21 @@ def _determine_account(config): :raises errors.Error: If unable to register an account with ACME server """ + def _tos_cb(terms_of_service): + if config.tos: + return True + msg = ("Please read the Terms of Service at {0}. You " + "must agree in order to register with the ACME " + "server at {1}".format( + terms_of_service, config.server)) + obj = zope.component.getUtility(interfaces.IDisplay) + result = obj.yesno(msg, "Agree", "Cancel", + cli_flag="--agree-tos", force_interactive=True) + if not result: + raise errors.Error( + "Registration cannot proceed without accepting " + "Terms of Service.") + account_storage = account.AccountFileStorage(config) acme = None @@ -501,21 +516,6 @@ def _determine_account(config): else: # no account registered yet if config.email is None and not config.register_unsafely_without_email: config.email = display_ops.get_email() - - def _tos_cb(terms_of_service): - if config.tos: - return True - msg = ("Please read the Terms of Service at {0}. You " - "must agree in order to register with the ACME " - "server at {1}".format( - terms_of_service, config.server)) - obj = zope.component.getUtility(interfaces.IDisplay) - result = obj.yesno(msg, "Agree", "Cancel", - cli_flag="--agree-tos", force_interactive=True) - if not result: - raise errors.Error( - "Registration cannot proceed without accepting " - "Terms of Service.") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) @@ -735,8 +735,9 @@ def register(config, unused_plugins): acc, acme = _determine_account(config) cb_client = client.Client(config, acc, None, None, acme=acme) # We rely on an exception to interrupt this process if it didn't work. + acc_contacts = ['mailto:' + email for email in config.email.split(',')] acc.regr = cb_client.acme.update_registration(acc.regr.update( - body=acc.regr.body.update(contact=('mailto:' + config.email,)))) + body=acc.regr.body.update(contact=acc_contacts))) account_storage.save_regr(acc, cb_client.acme) eff.handle_subscription(config) add_msg("Your e-mail address was updated to {0}.".format(config.email)) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 14cde27ee..8c9e24354 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1435,7 +1435,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mocked_storage = mock.MagicMock() mocked_account.AccountFileStorage.return_value = mocked_storage mocked_storage.find_all.return_value = ["an account"] - mocked_det.return_value = (mock.MagicMock(), "foo") + mock_acc = mock.MagicMock() + mock_regr = mock_acc.regr + mocked_det.return_value = (mock_acc, "foo") cb_client = mock.MagicMock() mocked_client.Client.return_value = cb_client x = self._call_no_clientmock( @@ -1445,8 +1447,10 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(x[0] is None) # and we got supposedly did update the registration from # the server - self.assertTrue( - cb_client.acme.update_registration.called) + reg_arg = cb_client.acme.update_registration.call_args[0][0] + # Test the return value of .update() was used because + # the regr is immutable. + self.assertEqual(reg_arg, mock_regr.update()) # and we saved the updated registration on disk self.assertTrue(mocked_storage.save_regr.called) self.assertTrue( diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 9748befa3..e931e30f3 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -191,7 +191,14 @@ for dir in $renewal_hooks_dirs; do exit 1 fi done -common register --update-registration --email example@example.org + +common unregister + +common register --email ex1@domain.org,ex2@domain.org + +common register --update-registration --email ex1@domain.org + +common register --update-registration --email ex1@domain.org,ex2@domain.org common plugins --init --prepare | grep webroot From 8440d0814de346fb8beb2ca1497e1cc7803a19fe Mon Sep 17 00:00:00 2001 From: pdamodaran Date: Tue, 22 May 2018 18:35:12 -0400 Subject: [PATCH 68/93] fixed dependency-requirements.txt (#6023) --- letsencrypt-auto-source/letsencrypt-auto | 18 ++++++++++++------ .../pieces/dependency-requirements.txt | 18 ++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0b83b08a7..28281e20d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1055,9 +1055,11 @@ cffi==1.10.0 \ --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 + --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ + --no-binary ConfigArgParse configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ + --no-binary configobj cryptography==2.0.2 \ --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ @@ -1112,7 +1114,8 @@ mock==1.3.0 \ --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ + --no-binary ordereddict packaging==16.8 \ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e @@ -1138,7 +1141,8 @@ pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ + --no-binary python-augeas pytz==2015.7 \ --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ @@ -1166,9 +1170,11 @@ unittest2==1.1.0 \ --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ + --no-binary zope.component zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ + --no-binary zope.event zope.interface==4.1.3 \ --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index a30a32b48..376e19deb 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -59,9 +59,11 @@ cffi==1.10.0 \ --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 ConfigArgParse==0.12.0 \ - --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 + --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ + --no-binary ConfigArgParse configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ + --no-binary configobj cryptography==2.0.2 \ --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \ --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \ @@ -116,7 +118,8 @@ mock==1.3.0 \ --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ - --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \ + --no-binary ordereddict packaging==16.8 \ --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \ --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e @@ -142,7 +145,8 @@ pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \ + --no-binary python-augeas pytz==2015.7 \ --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ @@ -170,9 +174,11 @@ unittest2==1.1.0 \ --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 zope.component==4.2.2 \ - --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \ + --no-binary zope.component zope.event==4.1.0 \ - --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \ + --no-binary zope.event zope.interface==4.1.3 \ --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ From deb5b072d9bfb10646df5a7de71f05ac6c9b9cd6 Mon Sep 17 00:00:00 2001 From: Kevin Le Date: Wed, 23 May 2018 12:59:49 -0400 Subject: [PATCH 69/93] Log cases when standalone fails to bind a port. (#5985) * Log cases when standalone fails to bind a port. * Fix linter + changed log message * Changed multiline string format * Fixed indentation in standalone.py --- acme/acme/standalone.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index a370501ee..ff9159933 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -83,9 +83,22 @@ class BaseDualNetworkedServers(object): new_address = (server_address[0],) + (port,) + server_address[2:] new_args = (new_address,) + remaining_args server = ServerClass(*new_args, **kwargs) # pylint: disable=star-args - except socket.error: - logger.debug("Failed to bind to %s:%s using %s", new_address[0], + logger.debug( + "Successfully bound to %s:%s using %s", new_address[0], new_address[1], "IPv6" if ip_version else "IPv4") + except socket.error: + if self.servers: + # Already bound using IPv6. + logger.debug( + "Certbot wasn't able to bind to %s:%s using %s, this " + + "is often expected due to the dual stack nature of " + + "IPv6 socket implementations.", + new_address[0], new_address[1], + "IPv6" if ip_version else "IPv4") + else: + logger.debug( + "Failed to bind to %s:%s using %s", new_address[0], + new_address[1], "IPv6" if ip_version else "IPv4") else: self.servers.append(server) # If two servers are set up and port 0 was passed in, ensure we always From 4304ff0d623bc3512343c09f545579119887138e Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 23 May 2018 11:33:21 -0700 Subject: [PATCH 70/93] Bring up just the boulder container. (#6031) Boulder recently added a "netaccess" container which may conflict. --- tests/boulder-fetch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index d513ec064..7a677ad12 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -17,7 +17,7 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml -docker-compose up -d +docker-compose up -d boulder set +x # reduce verbosity while waiting for boulder until curl http://localhost:4000/directory 2>/dev/null; do From 0b215366b1c66950195ddbbfb6f16b8657f4b198 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 May 2018 13:57:22 -0700 Subject: [PATCH 71/93] turn off cancel notifications (#5918) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 111ddb3d4..e3d964326 100644 --- a/.travis.yml +++ b/.travis.yml @@ -107,6 +107,7 @@ notifications: irc: channels: - secure: "SGWZl3ownKx9xKVV2VnGt7DqkTmutJ89oJV9tjKhSs84kLijU6EYdPnllqISpfHMTxXflNZuxtGo0wTDYHXBuZL47w1O32W6nzuXdra5zC+i4sYQwYULUsyfOv9gJX8zWAULiK0Z3r0oho45U+FR5ZN6TPCidi8/eGU+EEPwaAw=" + on_cancel: never on_success: never on_failure: always use_notice: true From a1f5dc27f28e46cc2aa92777f12f17b418fd6e7c Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 23 May 2018 14:03:30 -0700 Subject: [PATCH 72/93] Add domain to error message when no matching server block found (#6034) --- certbot-nginx/certbot_nginx/configurator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 118699aa2..293f7378e 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -334,7 +334,7 @@ class NginxConfigurator(common.Installer): def _vhost_from_duplicated_default(self, domain, port=None): if self.new_vhost is None: - default_vhost = self._get_default_vhost(port) + default_vhost = self._get_default_vhost(port, domain) self.new_vhost = self.parser.duplicate_vhost(default_vhost, remove_singleton_listen_params=True) self.new_vhost.names = set() @@ -350,7 +350,7 @@ class NginxConfigurator(common.Installer): name_block[0].append(name) self.parser.update_or_add_server_directives(vhost, name_block) - def _get_default_vhost(self, port): + def _get_default_vhost(self, port, domain): vhost_list = self.parser.get_vhosts() # if one has default_server set, return that one default_vhosts = [] @@ -367,7 +367,7 @@ class NginxConfigurator(common.Installer): # TODO: present a list of vhosts for user to choose from raise errors.MisconfigurationError("Could not automatically find a matching server" - " block. Set the `server_name` directive to use the Nginx installer.") + " block for %s. Set the `server_name` directive to use the Nginx installer." % domain) def _get_ranked_matches(self, target_name): """Returns a ranked list of vhosts that match target_name. From b1bcccb04bbd4d9d35bc95397f9ba1633b91242a Mon Sep 17 00:00:00 2001 From: Jeremy Gillula Date: Wed, 23 May 2018 20:40:34 -0700 Subject: [PATCH 73/93] Changing opt-in ask for emails (#6035) --- certbot/eff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/eff.py b/certbot/eff.py index b047c0b97..388ae986b 100644 --- a/certbot/eff.py +++ b/certbot/eff.py @@ -41,8 +41,8 @@ def _want_subscription(): 'Would you be willing to share your email address with the ' "Electronic Frontier Foundation, a founding partner of the Let's " 'Encrypt project and the non-profit organization that develops ' - "Certbot? We'd like to send you email about EFF and our work to " - 'encrypt the web, protect its users and defend digital rights.') + "Certbot? We'd like to send you email about our work encrypting " + "the web, EFF news, campaigns, and ways to support digital freedom. ") display = zope.component.getUtility(interfaces.IDisplay) return display.yesno(prompt, default=False) From a03c68fc8334cac84e8ee22f2b7b7fa7fc323ab9 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 24 May 2018 10:53:21 -0700 Subject: [PATCH 74/93] Clean up boulder-fetch a bit. (#6032) The value for FAKE_DNS is now always the same because Boulder's docker-compose hardcodes it, so skip some sed. Set a time limit on how long we'll wait for boulder to come up. --- tests/boulder-fetch.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 7a677ad12..31e0f6b30 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -11,16 +11,20 @@ if [ ! -d ${BOULDERPATH} ]; then fi cd ${BOULDERPATH} -FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') -[ -z "$FAKE_DNS" ] && FAKE_DNS=$(ifconfig docker0 | grep "inet " | xargs | cut -d ' ' -f 2) -[ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) -[ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 -sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml +sed -i "s/FAKE_DNS: .*/FAKE_DNS: 10.77.77.1/" docker-compose.yml docker-compose up -d boulder set +x # reduce verbosity while waiting for boulder -until curl http://localhost:4000/directory 2>/dev/null; do - echo waiting for boulder - sleep 1 +for n in `seq 1 150` ; do + if curl http://localhost:4000/directory 2>/dev/null; then + break + else + sleep 1 + fi done + +if ! curl http://localhost:4000/directory 2>/dev/null; then + echo "timed out waiting for boulder to start" + exit 1 +fi From e48c653245bc08b7e517465aea32f678c5b9b64b Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 25 May 2018 21:00:37 +0300 Subject: [PATCH 75/93] Change GenericUpdater parameter to lineage (#6030) In order to give more flexibility for plugins using interfaces.GenericUpdater interface, lineage needs to be passed to the updater method instead of individual domains. All of the (present and potential) installers do not work on per domain basis, while the lineage does contain a list of them for installers which do. This also means that we don't unnecessarily run the updater method multiple times, potentially invoking expensive tooling up to $max_san_amount times. * Make GenericUpdater use lineage as parameter and get invoked only once per lineage --- certbot/interfaces.py | 18 +++++++++--------- certbot/tests/renewupdater_test.py | 11 ++++------- certbot/updater.py | 7 +++---- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 6233e3592..a5fb426e6 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -604,10 +604,10 @@ class IReporter(zope.interface.Interface): # When "certbot renew" is run, Certbot will iterate over each lineage and check # if the selected installer for that lineage is a subclass of each updater # class. If it is and the update of that type is configured to be run for that -# lineage, the relevant update function will be called for each domain in the -# lineage. These functions are never called for other subcommands, so if an -# installer wants to perform an update during the run or install subcommand, it -# should do so when :func:`IInstaller.deploy_cert` is called. +# lineage, the relevant update function will be called for it. These functions +# are never called for other subcommands, so if an installer wants to perform +# an update during the run or install subcommand, it should do so when +# :func:`IInstaller.deploy_cert` is called. @six.add_metaclass(abc.ABCMeta) class GenericUpdater(object): @@ -623,7 +623,7 @@ class GenericUpdater(object): """ @abc.abstractmethod - def generic_updates(self, domain, *args, **kwargs): + def generic_updates(self, lineage, *args, **kwargs): """Perform any update types defined by the installer. If an installer is a subclass of the class containing this method, this @@ -631,9 +631,10 @@ class GenericUpdater(object): update defined by the installer should be run conditionally, the installer needs to handle checking the conditions itself. - This method is called once for each domain. + This method is called once for each lineage. - :param str domain: domain to handle the updates for + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert """ @@ -661,8 +662,7 @@ class RenewDeployer(object): This method is called once for each lineage renewed - :param lineage: Certificate lineage object that is set if certificate - was renewed on this run. + :param lineage: Certificate lineage object :type lineage: storage.RenewableCert """ diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index 9d0f8d515..ade8db390 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -19,7 +19,7 @@ class RenewUpdaterTest(unittest.TestCase): # pylint: disable=unused-argument self.restart = mock.MagicMock() self.callcounter = mock.MagicMock() - def generic_updates(self, domain, *args, **kwargs): + def generic_updates(self, lineage, *args, **kwargs): self.callcounter(*args, **kwargs) class MockInstallerRenewDeployer(interfaces.RenewDeployer): @@ -46,9 +46,7 @@ class RenewUpdaterTest(unittest.TestCase): def test_server_updates(self, _, mock_select, mock_getsave): config = self.get_config({"disable_renew_updates": False}) - lineage = mock.MagicMock() - lineage.names.return_value = ['firstdomain', 'seconddomain'] - mock_getsave.return_value = lineage + mock_getsave.return_value = mock.MagicMock() mock_generic_updater = self.generic_updater # Generic Updater @@ -59,14 +57,13 @@ class RenewUpdaterTest(unittest.TestCase): mock_generic_updater.restart.reset_mock() mock_generic_updater.callcounter.reset_mock() - updater.run_generic_updaters(config, None, lineage) - self.assertEqual(mock_generic_updater.callcounter.call_count, 2) + updater.run_generic_updaters(config, None, mock.MagicMock()) + self.assertEqual(mock_generic_updater.callcounter.call_count, 1) self.assertFalse(mock_generic_updater.restart.called) def test_renew_deployer(self): config = self.get_config({"disable_renew_updates": False}) lineage = mock.MagicMock() - lineage.names.return_value = ['firstdomain', 'seconddomain'] mock_deployer = self.renew_deployer updater.run_renewal_deployer(lineage, mock_deployer, config) self.assertTrue(mock_deployer.callcounter.called_with(lineage)) diff --git a/certbot/updater.py b/certbot/updater.py index f822c55ee..1216f38a6 100644 --- a/certbot/updater.py +++ b/certbot/updater.py @@ -61,7 +61,6 @@ def _run_updaters(lineage, installer, config): :returns: `None` :rtype: None """ - for domain in lineage.names(): - if not config.disable_renew_updates: - if isinstance(installer, interfaces.GenericUpdater): - installer.generic_updates(domain) + if not config.disable_renew_updates: + if isinstance(installer, interfaces.GenericUpdater): + installer.generic_updates(lineage) From 9f6b147d6f06e534351377195b4f28c61270115d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Sat, 26 May 2018 18:31:23 +0300 Subject: [PATCH 76/93] Do not call updaters and deployers when run with --dry-run (#6038) When Certbot is run with --dry-run, skip running GenericUpdater and RenewDeployer interface methods. This PR also makes the parameter order of updater.run_generic_updaters and updater.run_renewal_deployer consistent. Fixes #5927 * Do not call updaters and deployers when run with --dry-run * Use ConfigTestCase instead of mocking config objects manually --- certbot/main.py | 2 +- certbot/renewal.py | 4 ++-- certbot/tests/main_test.py | 3 ++- certbot/tests/renewupdater_test.py | 35 ++++++++++++++++++------------ certbot/updater.py | 20 ++++++++++++----- 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index dad1b793e..8fa6ebfc6 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1163,7 +1163,7 @@ def renew_cert(config, plugins, lineage): # In case of a renewal, reload server to pick up new certificate. # In principle we could have a configuration option to inhibit this # from happening. - updater.run_renewal_deployer(renewed_lineage, installer, config) + updater.run_renewal_deployer(config, renewed_lineage, installer) installer.restart() notify("new certificate deployed with reload of {0} server; fullchain is {1}".format( config.installer, lineage.fullchain), pause=False) diff --git a/certbot/renewal.py b/certbot/renewal.py index 0a6568426..236330a85 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -431,8 +431,8 @@ def handle_renewal_request(config): renew_skipped.append("%s expires on %s" % (renewal_candidate.fullchain, expiry.strftime("%Y-%m-%d"))) # Run updater interface methods - updater.run_generic_updaters(lineage_config, plugins, - renewal_candidate) + updater.run_generic_updaters(lineage_config, renewal_candidate, + plugins) except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 8c9e24354..36821cf53 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1464,7 +1464,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met None, None, None) with mock.patch('certbot.updater.logger.warning') as mock_log: - updater.run_generic_updaters(None, None, None) + self.config.dry_run = False + updater.run_generic_updaters(self.config, None, None) self.assertTrue(mock_log.called) self.assertTrue("Could not choose appropriate plugin for updaters" in mock_log.call_args[0][0]) diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index ade8db390..bd1cd891e 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -9,10 +9,11 @@ from certbot import updater import certbot.tests.util as test_util -class RenewUpdaterTest(unittest.TestCase): +class RenewUpdaterTest(test_util.ConfigTestCase): """Tests for interfaces.RenewDeployer and interfaces.GenericUpdater""" def setUp(self): + super(RenewUpdaterTest, self).setUp() class MockInstallerGenericUpdater(interfaces.GenericUpdater): """Mock class that implements GenericUpdater""" def __init__(self, *args, **kwargs): @@ -33,41 +34,47 @@ class RenewUpdaterTest(unittest.TestCase): self.generic_updater = MockInstallerGenericUpdater() self.renew_deployer = MockInstallerRenewDeployer() - def get_config(self, args): - """Get mock config from dict of parameters""" - config = mock.MagicMock() - for key in args.keys(): - config.__dict__[key] = args[key] - return config - @mock.patch('certbot.main._get_and_save_cert') @mock.patch('certbot.plugins.selection.choose_configurator_plugins') @test_util.patch_get_utility() def test_server_updates(self, _, mock_select, mock_getsave): - config = self.get_config({"disable_renew_updates": False}) - mock_getsave.return_value = mock.MagicMock() mock_generic_updater = self.generic_updater # Generic Updater mock_select.return_value = (mock_generic_updater, None) with mock.patch('certbot.main._init_le_client'): - main.renew_cert(config, None, mock.MagicMock()) + main.renew_cert(self.config, None, mock.MagicMock()) self.assertTrue(mock_generic_updater.restart.called) mock_generic_updater.restart.reset_mock() mock_generic_updater.callcounter.reset_mock() - updater.run_generic_updaters(config, None, mock.MagicMock()) + updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertEqual(mock_generic_updater.callcounter.call_count, 1) self.assertFalse(mock_generic_updater.restart.called) def test_renew_deployer(self): - config = self.get_config({"disable_renew_updates": False}) lineage = mock.MagicMock() mock_deployer = self.renew_deployer - updater.run_renewal_deployer(lineage, mock_deployer, config) + updater.run_renewal_deployer(self.config, lineage, mock_deployer) self.assertTrue(mock_deployer.callcounter.called_with(lineage)) + @mock.patch("certbot.updater.logger.debug") + def test_updater_skip_dry_run(self, mock_log): + self.config.dry_run = True + updater.run_generic_updaters(self.config, None, None) + self.assertTrue(mock_log.called) + self.assertEquals(mock_log.call_args[0][0], + "Skipping updaters in dry-run mode.") + + @mock.patch("certbot.updater.logger.debug") + def test_deployer_skip_dry_run(self, mock_log): + self.config.dry_run = True + updater.run_renewal_deployer(self.config, None, None) + self.assertTrue(mock_log.called) + self.assertEquals(mock_log.call_args[0][0], + "Skipping renewal deployer in dry-run mode.") + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/updater.py b/certbot/updater.py index 1216f38a6..112cf06ef 100644 --- a/certbot/updater.py +++ b/certbot/updater.py @@ -8,21 +8,24 @@ from certbot.plugins import selection as plug_sel logger = logging.getLogger(__name__) -def run_generic_updaters(config, plugins, lineage): +def run_generic_updaters(config, lineage, plugins): """Run updaters that the plugin supports :param config: Configuration object :type config: interfaces.IConfig - :param plugins: List of plugins - :type plugins: `list` of `str` - :param lineage: Certificate lineage object :type lineage: storage.RenewableCert + :param plugins: List of plugins + :type plugins: `list` of `str` + :returns: `None` :rtype: None """ + if config.dry_run: + logger.debug("Skipping updaters in dry-run mode.") + return try: # installers are used in auth mode to determine domain names installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "certonly") @@ -31,10 +34,13 @@ def run_generic_updaters(config, plugins, lineage): return _run_updaters(lineage, installer, config) -def run_renewal_deployer(lineage, installer, config): +def run_renewal_deployer(config, lineage, installer): """Helper function to run deployer interface method if supported by the used installer plugin. + :param config: Configuration object + :type config: interfaces.IConfig + :param lineage: Certificate lineage object :type lineage: storage.RenewableCert @@ -44,6 +50,10 @@ def run_renewal_deployer(lineage, installer, config): :returns: `None` :rtype: None """ + if config.dry_run: + logger.debug("Skipping renewal deployer in dry-run mode.") + return + if not config.disable_renew_updates and isinstance(installer, interfaces.RenewDeployer): installer.renew_deploy(lineage) From d53ef1f7c24c21d297a23349f3d566e24252b1fb Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 31 May 2018 13:57:23 -0700 Subject: [PATCH 77/93] Add mypy info to Certbot docs (#6033) * Add mypy info to Certbot docs * break up lines * link to mypy docs and links to https * Expand on import wording * be consistent about mypy styling --- docs/contributing.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index ed986c562..58db251d4 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -312,6 +312,40 @@ Please: .. _PEP 8 - Style Guide for Python Code: https://www.python.org/dev/peps/pep-0008 +Mypy type annotations +===================== + +Certbot uses the `mypy`_ static type checker. Python 3 natively supports official type annotations, +which can then be tested for consistency using mypy. Python 2 doesn’t, but type annotations can +be `added in comments`_. Mypy does some type checks even without type annotations; we can find +bugs in Certbot even without a fully annotated codebase. + +Certbot supports both Python 2 and 3, so we’re using Python 2-style annotations. + +Zulip wrote a `great guide`_ to using mypy. It’s useful, but you don’t have to read the whole thing +to start contributing to Certbot. + +To run mypy on Certbot, use ``tox -e mypy`` on a machine that has Python 3 installed. + +Note that instead of just importing ``typing``, due to packaging issues, in Certbot we import from +``acme.magic_typing`` and have to add some comments for pylint like this: + +.. code-block:: python + + from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module + +Also note that OpenSSL, which we rely on, has type definitions for crypto but not SSL. We use both. +Those imports should look like this: + +.. code-block:: python + + from OpenSSL import crypto + from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 + +.. _mypy: https://mypy.readthedocs.io +.. _added in comments: https://mypy.readthedocs.io/en/latest/cheat_sheet.html +.. _great guide: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/ + Submitting a pull request ========================= From fb0d2ec3d60cf29d2c8aff09a3adbf50ba7f938f Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Fri, 1 Jun 2018 18:09:02 -0400 Subject: [PATCH 78/93] Include missing space (#6061) --- certbot-apache/certbot_apache/configurator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index bb82a9d3f..861fe4458 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -132,10 +132,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): default=cls.OS_DEFAULTS["challenge_location"], help="Directory path for challenge configuration.") add("handle-modules", default=cls.OS_DEFAULTS["handle_mods"], - help="Let installer handle enabling required modules for you." + + help="Let installer handle enabling required modules for you. " + "(Only Ubuntu/Debian currently)") add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"], - help="Let installer handle enabling sites for you." + + help="Let installer handle enabling sites for you. " + "(Only Ubuntu/Debian currently)") util.add_deprecated_argument(add, argument_name="ctl", nargs=1) util.add_deprecated_argument( From e2d6faa8a95b05e11369f7ae646fa8216b2b883a Mon Sep 17 00:00:00 2001 From: schoen Date: Fri, 1 Jun 2018 15:21:02 -0700 Subject: [PATCH 79/93] Add --reuse-key feature (#5901) * Initial work on new version of --reuse-key * Test for reuse_key * Make lint happier * Also test a non-dry-run reuse_key renewal * Test --reuse-key in boulder integration test * Better reuse-key integration testing * Log fact that key was reused * Test that the certificates themselves are different * Change "oldkeypath" to "old_keypath" * Simply appearance of new-key generation logic * Reorganize new-key logic * Move awk logic into TotalAndDistinctLines function * After refactor, there's now explicit None rather than missing param * Indicate for MyPy that key can be None * Actually import the Optional type * magic_typing is too magical for pylint * Remove --no-reuse-key option * Correct pylint test disable --- certbot/cli.py | 6 ++++++ certbot/client.py | 32 +++++++++++++++++++++++++++----- certbot/constants.py | 1 + certbot/renewal.py | 7 +++++-- certbot/tests/main_test.py | 24 +++++++++++++++++++++--- tests/boulder-integration.sh | 28 ++++++++++++++++++++++++++++ 6 files changed, 88 insertions(+), 10 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 25319bbd8..05e316133 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1017,6 +1017,12 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "certificate already exists for the requested certificate name " "but does not match the requested domains, renew it now, " "regardless of whether it is near expiry.") + helpful.add( + "automation", "--reuse-key", dest="reuse_key", + action="store_true", default=flag_default("reuse_key"), + help="When renewing, use the same private key as the existing " + "certificate.") + helpful.add( ["automation", "renew", "certonly"], "--allow-subset-of-names", action="store_true", diff --git a/certbot/client.py b/certbot/client.py index dadc3a0f8..cc2f31d56 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -4,6 +4,7 @@ import logging import os import platform + from cryptography.hazmat.backends import default_backend # https://github.com/python/typeshed/blob/master/third_party/ # 2/cryptography/hazmat/primitives/asymmetric/rsa.pyi @@ -16,6 +17,7 @@ from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors from acme import messages +from acme.magic_typing import Optional # pylint: disable=unused-import,no-name-in-module import certbot @@ -273,7 +275,7 @@ class Client(object): cert, chain = crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) return cert.encode(), chain.encode() - def obtain_certificate(self, domains): + def obtain_certificate(self, domains, old_keypath=None): """Obtains a certificate from the ACME server. `.register` must be called before `.obtain_certificate` @@ -286,16 +288,36 @@ class Client(object): :rtype: tuple """ + + # We need to determine the key path, key PEM data, CSR path, + # and CSR PEM data. For a dry run, the paths are None because + # they aren't permanently saved to disk. For a lineage with + # --reuse-key, the key path and PEM data are derived from an + # existing file. + + if old_keypath is not None: + # We've been asked to reuse a specific existing private key. + # Therefore, we'll read it now and not generate a new one in + # either case below. + with open(old_keypath, "r") as f: + keypath = old_keypath + keypem = f.read() + key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key] + logger.info("Reusing existing private key from %s.", old_keypath) + else: + # The key is set to None here but will be created below. + key = None + # Create CSR from names if self.config.dry_run: - key = util.Key(file=None, - pem=crypto_util.make_key(self.config.rsa_key_size)) + key = key or util.Key(file=None, + pem=crypto_util.make_key(self.config.rsa_key_size)) csr = util.CSR(file=None, form="pem", data=acme_crypto_util.make_csr( key.pem, domains, self.config.must_staple)) else: - key = crypto_util.init_save_key( - self.config.rsa_key_size, self.config.key_dir) + key = key or crypto_util.init_save_key(self.config.rsa_key_size, + self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) orderr = self._get_order_and_authorizations(csr.data, self.config.allow_subset_of_names) diff --git a/certbot/constants.py b/certbot/constants.py index 40557d287..1dd25e799 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -64,6 +64,7 @@ CLI_DEFAULTS = dict( pref_challs=[], validate_hooks=True, directory_hooks=True, + reuse_key=False, disable_renew_updates=False, # Subparsers diff --git a/certbot/renewal.py b/certbot/renewal.py index 236330a85..aa8c9722a 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -36,7 +36,7 @@ STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", "pre_hook", "post_hook", "tls_sni_01_address", "http01_address"] INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] -BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names"] +BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names", "reuse_key"] CONFIG_ITEMS = set(itertools.chain( BOOL_CONFIG_ITEMS, INT_CONFIG_ITEMS, STR_CONFIG_ITEMS, ('pref_challs',))) @@ -298,7 +298,10 @@ def renew_cert(config, domains, le_client, lineage): _avoid_invalidating_lineage(config, lineage, original_server) if not domains: domains = lineage.names() - new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains) + # The private key is the existing lineage private key if reuse_key is set. + # Otherwise, generate a fresh private key by passing None. + new_key = lineage.privkey if config.reuse_key else None + new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains, new_key) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert)) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 36821cf53..4b251c421 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1026,8 +1026,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, - quiet_mode=False, expiry_date=datetime.datetime.now()): - # pylint: disable=too-many-locals,too-many-arguments + quiet_mode=False, expiry_date=datetime.datetime.now(), + reuse_key=False): + # pylint: disable=too-many-locals,too-many-arguments,too-many-branches cert_path = test_util.vector_path('cert_512.pem') chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path, @@ -1077,7 +1078,13 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met traceback.format_exc()) if should_renew: - mock_client.obtain_certificate.assert_called_once_with(['isnot.org']) + if reuse_key: + # The location of the previous live privkey.pem is passed + # to obtain_certificate + mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], + os.path.join(self.config.config_dir, "live/sample-renewal/privkey.pem")) + else: + mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], None) else: self.assertEqual(mock_client.obtain_certificate.call_count, 0) except: @@ -1127,6 +1134,17 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, should_renew=True) + def test_reuse_key(self): + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + args = ["renew", "--dry-run", "--reuse-key"] + self._test_renewal_common(True, [], args=args, should_renew=True, reuse_key=True) + + @mock.patch('certbot.storage.RenewableCert.save_successor') + def test_reuse_key_no_dry_run(self, unused_save_successor): + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + args = ["renew", "--reuse-key"] + self._test_renewal_common(True, [], args=args, should_renew=True, reuse_key=True) + @mock.patch('certbot.renewal.should_renew') def test_renew_skips_recent_certs(self, should_renew): should_renew.return_value = False diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index e931e30f3..ef611e743 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -166,6 +166,14 @@ CheckRenewHook() { CheckSavedRenewHook $1 } +# Return success only if input contains exactly $1 lines of text, of +# which $2 different values occur in the first field. +TotalAndDistinctLines() { + total=$1 + distinct=$2 + awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' +} + # Cleanup coverage data coverage erase @@ -347,6 +355,26 @@ if common certificates | grep "fail\.dns1\.le\.wtf"; then exit 1 fi +# reuse-key +common --domains reusekey.le.wtf --reuse-key +common renew --cert-name reusekey.le.wtf +CheckCertCount "reusekey.le.wtf" 2 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# The final awk command here exits successfully if its input consists of +# exactly two lines with identical first fields, and unsuccessfully otherwise. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 2 1 + +# don't reuse key (just by forcing reissuance without --reuse-key) +common --cert-name reusekey.le.wtf --domains reusekey.le.wtf --force-renewal +CheckCertCount "reusekey.le.wtf" 3 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# Exactly three lines, of which exactly two identical first fields. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 3 2 + +# Nonetheless, all three certificates are different even though two of them +# share the same subject key. +sha256sum "${root}/conf/archive/reusekey.le.wtf/cert"* | TotalAndDistinctLines 3 3 + # ECDSA openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ From f19ebab44173b3ed70eb2aa81464fe1aa2fc0122 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 4 Jun 2018 21:08:40 +0300 Subject: [PATCH 80/93] Make sure the pluginstorage file gets truncated when writing to it (#6062) --- certbot/plugins/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot/plugins/storage.py b/certbot/plugins/storage.py index 9472a1ebb..ae3ca1889 100644 --- a/certbot/plugins/storage.py +++ b/certbot/plugins/storage.py @@ -84,7 +84,8 @@ class PluginStorage(object): raise errors.PluginStorageError(errmsg) try: with os.fdopen(os.open(self._storagepath, - os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fh: + os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + 0o600), 'w') as fh: fh.write(serialized) except IOError as e: errmsg = "Could not write PluginStorage data to file {0} : {1}".format( From 4151737e171b9a73923d02dc746cb593fb1f6d33 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 4 Jun 2018 13:13:23 -0700 Subject: [PATCH 81/93] Read in bytes to fix --reuse-key on Python 3 (#6069) * Read bytes for now for compatibility * add clarifying comment --- certbot/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/certbot/client.py b/certbot/client.py index cc2f31d56..d97de0571 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -299,7 +299,10 @@ class Client(object): # We've been asked to reuse a specific existing private key. # Therefore, we'll read it now and not generate a new one in # either case below. - with open(old_keypath, "r") as f: + # + # We read in bytes here because the type of `key.pem` + # created below is also bytes. + with open(old_keypath, "rb") as f: keypath = old_keypath keypem = f.read() key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key] From 15f1405fff7083bf5d4f599a58c54a43be499740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20D=C4=99bski?= Date: Mon, 4 Jun 2018 23:54:17 +0200 Subject: [PATCH 82/93] Implement TLS-ALPN-01 challenge and standalone TLS-ALPN server (#5894) The new challenge is described in https://github.com/rolandshoemaker/acme-tls-alpn. * TLS-ALPN tests * Implement TLS-ALPN challenge * Skip TLS-ALPN tests on old pyopenssl * make _selection methods private. --- acme/acme/challenges.py | 152 +++++++++++++++++++++++++++- acme/acme/challenges_test.py | 121 ++++++++++++++++++++++ acme/acme/crypto_util.py | 61 ++++++++--- acme/acme/crypto_util_test.py | 16 ++- acme/acme/standalone.py | 48 ++++++++- acme/acme/standalone_test.py | 57 +++++++++++ acme/acme/testdata/README | 6 +- acme/acme/testdata/rsa1024_cert.pem | 13 +++ 8 files changed, 453 insertions(+), 21 deletions(-) create mode 100644 acme/acme/testdata/rsa1024_cert.pem diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 674f2c38f..ce788e2cc 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -1,5 +1,6 @@ """ACME Identifier Validation Challenges.""" import abc +import codecs import functools import hashlib import logging @@ -7,7 +8,7 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose -import OpenSSL +from OpenSSL import crypto import requests import six @@ -411,8 +412,8 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): """ if key is None: - key = OpenSSL.crypto.PKey() - key.generate_key(OpenSSL.crypto.TYPE_RSA, bits) + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) return crypto_util.gen_ss_cert(key, [ # z_domain is too big to fit into CN, hence first dummy domain 'dummy', self.z_domain.decode()], force_san=True), key @@ -507,6 +508,151 @@ class TLSSNI01(KeyAuthorizationChallenge): return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) +@ChallengeResponse.register +class TLSALPN01Response(KeyAuthorizationChallengeResponse): + """ACME tls-alpn-01 challenge response.""" + typ = "tls-alpn-01" + + PORT = 443 + """Verification port as defined by the protocol. + + You can override it (e.g. for testing) by passing ``port`` to + `simple_verify`. + + """ + + ID_PE_ACME_IDENTIFIER_V1 = b"1.3.6.1.5.5.7.1.30.1" + ACME_TLS_1_PROTOCOL = "acme-tls/1" + + @property + def h(self): + """Hash value stored in challenge certificate""" + return hashlib.sha256(self.key_authorization.encode('utf-8')).digest() + + def gen_cert(self, domain, key=None, bits=2048): + """Generate tls-alpn-01 certificate. + + :param unicode domain: Domain verified by the challenge. + :param OpenSSL.crypto.PKey key: Optional private key used in + certificate generation. If not provided (``None``), then + fresh key will be generated. + :param int bits: Number of bits for newly generated key. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + if key is None: + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) + + + der_value = b"DER:" + codecs.encode(self.h, 'hex') + acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1, + critical=True, value=der_value) + + return crypto_util.gen_ss_cert(key, [domain], force_san=True, + extensions=[acme_extension]), key + + def probe_cert(self, domain, host=None, port=None): + """Probe tls-alpn-01 challenge certificate. + + :param unicode domain: domain being validated, required. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + """ + if host is None: + host = socket.gethostbyname(domain) + logger.debug('%s resolved to %s', domain, host) + if port is None: + port = self.PORT + + return crypto_util.probe_sni(host=host, port=port, name=domain, + alpn_protocols=[self.ACME_TLS_1_PROTOCOL]) + + def verify_cert(self, domain, cert): + """Verify tls-alpn-01 challenge certificate. + + :param unicode domain: Domain name being validated. + :param OpensSSL.crypto.X509 cert: Challenge certificate. + + :returns: Whether the certificate was successfully verified. + :rtype: bool + + """ + # pylint: disable=protected-access + names = crypto_util._pyopenssl_cert_or_req_all_names(cert) + logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), names) + if len(names) != 1 or names[0].lower() != domain.lower(): + return False + + for i in range(cert.get_extension_count()): + ext = cert.get_extension(i) + # FIXME: assume this is the ACME extension. Currently there is no + # way to get full OID of an unknown extension from pyopenssl. + if ext.get_short_name() == b'UNDEF': + data = ext.get_data() + return data == self.h + + return False + + # pylint: disable=too-many-arguments + def simple_verify(self, chall, domain, account_public_key, + cert=None, host=None, port=None): + """Simple verify. + + Verify ``validation`` using ``account_public_key``, optionally + probe tls-alpn-01 certificate and check using `verify_cert`. + + :param .challenges.TLSALPN01 chall: Corresponding challenge. + :param str domain: Domain name being validated. + :param JWK account_public_key: + :param OpenSSL.crypto.X509 cert: Optional certificate. If not + provided (``None``) certificate will be retrieved using + `probe_cert`. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + + :returns: ``True`` iff client's control of the domain has been + verified. + :rtype: bool + + """ + if not self.verify(chall, account_public_key): + logger.debug("Verification of key authorization in response failed") + return False + + if cert is None: + try: + cert = self.probe_cert(domain=domain, host=host, port=port) + except errors.Error as error: + logger.debug(str(error), exc_info=True) + return False + + return self.verify_cert(cert, domain) + + +@Challenge.register # pylint: disable=too-many-ancestors +class TLSALPN01(KeyAuthorizationChallenge): + """ACME tls-alpn-01 challenge.""" + response_cls = TLSALPN01Response + typ = response_cls.typ + + def validation(self, account_key, **kwargs): + """Generate validation. + + :param JWK account_key: + :param OpenSSL.crypto.PKey cert_key: Optional private key used + in certificate generation. If not provided (``None``), then + fresh key will be generated. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + return self.response(account_key).gen_cert(key=kwargs.get('cert_key')) + + @Challenge.register # pylint: disable=too-many-ancestors class DNS(_TokenChallenge): """ACME "dns" challenge.""" diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 834d569aa..b929d4939 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -393,6 +393,127 @@ class TLSSNI01Test(unittest.TestCase): mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) +class TLSALPN01ResponseTest(unittest.TestCase): + # pylint: disable=too-many-instance-attributes + + def setUp(self): + from acme.challenges import TLSALPN01 + self.chall = TLSALPN01( + token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e')) + self.domain = u'example.com' + self.domain2 = u'example2.com' + + self.response = self.chall.response(KEY) + self.jmsg = { + 'resource': 'challenge', + 'type': 'tls-alpn-01', + 'keyAuthorization': self.response.key_authorization, + } + + def test_to_partial_json(self): + self.assertEqual(self.jmsg, self.response.to_partial_json()) + + def test_from_json(self): + from acme.challenges import TLSALPN01Response + self.assertEqual(self.response, TLSALPN01Response.from_json(self.jmsg)) + + def test_from_json_hashable(self): + from acme.challenges import TLSALPN01Response + hash(TLSALPN01Response.from_json(self.jmsg)) + + def test_gen_verify_cert(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_gen_verify_cert_gen_key(self): + cert, key = self.response.gen_cert(self.domain) + self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_verify_bad_cert(self): + self.assertFalse(self.response.verify_cert(self.domain, + test_util.load_cert('cert.pem'))) + + def test_verify_bad_domain(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertFalse(self.response.verify_cert(self.domain2, cert)) + + def test_simple_verify_bad_key_authorization(self): + key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) + self.response.simple_verify(self.chall, "local", key2.public_key()) + + @mock.patch('acme.challenges.TLSALPN01Response.verify_cert', autospec=True) + def test_simple_verify(self, mock_verify_cert): + mock_verify_cert.return_value = mock.sentinel.verification + self.assertEqual( + mock.sentinel.verification, self.response.simple_verify( + self.chall, self.domain, KEY.public_key(), + cert=mock.sentinel.cert)) + mock_verify_cert.assert_called_once_with( + self.response, mock.sentinel.cert, self.domain) + + @mock.patch('acme.challenges.socket.gethostbyname') + @mock.patch('acme.challenges.crypto_util.probe_sni') + def test_probe_cert(self, mock_probe_sni, mock_gethostbyname): + mock_gethostbyname.return_value = '127.0.0.1' + self.response.probe_cert('foo.com') + mock_gethostbyname.assert_called_once_with('foo.com') + mock_probe_sni.assert_called_once_with( + host='127.0.0.1', port=self.response.PORT, name='foo.com', + alpn_protocols=['acme-tls/1']) + + self.response.probe_cert('foo.com', host='8.8.8.8') + mock_probe_sni.assert_called_with( + host='8.8.8.8', port=mock.ANY, name='foo.com', + alpn_protocols=['acme-tls/1']) + + @mock.patch('acme.challenges.TLSALPN01Response.probe_cert') + def test_simple_verify_false_on_probe_error(self, mock_probe_cert): + mock_probe_cert.side_effect = errors.Error + self.assertFalse(self.response.simple_verify( + self.chall, self.domain, KEY.public_key())) + + +class TLSALPN01Test(unittest.TestCase): + + def setUp(self): + from acme.challenges import TLSALPN01 + self.msg = TLSALPN01( + token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e')) + self.jmsg = { + 'type': 'tls-alpn-01', + 'token': 'a82d5ff8ef740d12881f6d3c2277ab2e', + } + + def test_to_partial_json(self): + self.assertEqual(self.jmsg, self.msg.to_partial_json()) + + def test_from_json(self): + from acme.challenges import TLSALPN01 + self.assertEqual(self.msg, TLSALPN01.from_json(self.jmsg)) + + def test_from_json_hashable(self): + from acme.challenges import TLSALPN01 + hash(TLSALPN01.from_json(self.jmsg)) + + def test_from_json_invalid_token_length(self): + from acme.challenges import TLSALPN01 + self.jmsg['token'] = jose.encode_b64jose(b'abcd') + self.assertRaises( + jose.DeserializationError, TLSALPN01.from_json, self.jmsg) + + @mock.patch('acme.challenges.TLSALPN01Response.gen_cert') + def test_validation(self, mock_gen_cert): + mock_gen_cert.return_value = ('cert', 'key') + self.assertEqual(('cert', 'key'), self.msg.validation( + KEY, cert_key=mock.sentinel.cert_key)) + mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) + + class DNSTest(unittest.TestCase): def setUp(self): diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index d0e203417..d25c2340b 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -31,6 +31,15 @@ logger = logging.getLogger(__name__) _DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore +class _DefaultCertSelection(object): + def __init__(self, certs): + self.certs = certs + + def __call__(self, connection): + server_name = connection.get_servername() + return self.certs.get(server_name, None) + + class SSLSocket(object): # pylint: disable=too-few-public-methods """SSL wrapper for sockets. @@ -38,12 +47,25 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :ivar dict certs: Mapping from domain names (`bytes`) to `OpenSSL.crypto.X509`. :ivar method: See `OpenSSL.SSL.Context` for allowed values. + :ivar alpn_selection: Hook to select negotiated ALPN protocol for + connection. + :ivar cert_selection: Hook to select certificate for connection. If given, + `certs` parameter would be ignored, and therefore must be empty. """ - def __init__(self, sock, certs, method=_DEFAULT_TLSSNI01_SSL_METHOD): + def __init__(self, sock, certs=None, + method=_DEFAULT_TLSSNI01_SSL_METHOD, alpn_selection=None, + cert_selection=None): self.sock = sock - self.certs = certs + self.alpn_selection = alpn_selection self.method = method + if not cert_selection and not certs: + raise ValueError("Neither cert_selection or certs specified.") + if cert_selection and certs: + raise ValueError("Both cert_selection and certs specified.") + if cert_selection is None: + cert_selection = _DefaultCertSelection(certs) + self.cert_selection = cert_selection def __getattr__(self, name): return getattr(self.sock, name) @@ -60,18 +82,19 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods :type connection: :class:`OpenSSL.Connection` """ - server_name = connection.get_servername() - try: - key, cert = self.certs[server_name] - except KeyError: - logger.debug("Server name (%s) not recognized, dropping SSL", - server_name) + pair = self.cert_selection(connection) + if pair is None: + logger.debug("Certificate selection for server name %s failed, dropping SSL", + connection.get_servername()) return + key, cert = pair new_context = SSL.Context(self.method) new_context.set_options(SSL.OP_NO_SSLv2) new_context.set_options(SSL.OP_NO_SSLv3) new_context.use_privatekey(key) new_context.use_certificate(cert) + if self.alpn_selection is not None: + new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) class FakeConnection(object): @@ -96,6 +119,8 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods context.set_options(SSL.OP_NO_SSLv2) context.set_options(SSL.OP_NO_SSLv3) context.set_tlsext_servername_callback(self._pick_certificate_cb) + if self.alpn_selection is not None: + context.set_alpn_select_callback(self.alpn_selection) ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) ssl_sock.set_accept_state() @@ -111,8 +136,9 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods return ssl_sock, addr -def probe_sni(name, host, port=443, timeout=300, - method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0)): +def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-arguments + method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0), + alpn_protocols=None): """Probe SNI server for SSL certificate. :param bytes name: Byte string to send as the server name in the @@ -124,6 +150,8 @@ def probe_sni(name, host, port=443, timeout=300, :param tuple source_address: Enables multi-path probing (selection of source interface). See `socket.creation_connection` for more info. Available only in Python 2.7+. + :param alpn_protocols: Protocols to request using ALPN. + :type alpn_protocols: `list` of `bytes` :raises acme.errors.Error: In case of any problems. @@ -160,6 +188,8 @@ def probe_sni(name, host, port=443, timeout=300, client_ssl = SSL.Connection(context, client) client_ssl.set_connect_state() client_ssl.set_tlsext_host_name(name) # pyOpenSSL>=0.13 + if alpn_protocols is not None: + client_ssl.set_alpn_protos(alpn_protocols) try: client_ssl.do_handshake() client_ssl.shutdown() @@ -251,12 +281,14 @@ def _pyopenssl_cert_or_req_san(cert_or_req): def gen_ss_cert(key, domains, not_before=None, - validity=(7 * 24 * 60 * 60), force_san=True): + validity=(7 * 24 * 60 * 60), force_san=True, extensions=None): """Generate new self-signed certificate. :type domains: `list` of `unicode` :param OpenSSL.crypto.PKey key: :param bool force_san: + :param extensions: List of additional extensions to include in the cert. + :type extensions: `list` of `OpenSSL.crypto.X509Extension` If more than one domain is provided, all of the domains are put into ``subjectAltName`` X.509 extension and first domain is set as the @@ -269,10 +301,13 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16)) cert.set_version(2) - extensions = [ + if extensions is None: + extensions = [] + + extensions.append( crypto.X509Extension( b"basicConstraints", True, b"CA:TRUE, pathlen:0"), - ] + ) cert.get_subject().CN = domains[0] # TODO: what to put into cert.get_subject()? diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 36d62b324..b661e4e70 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -19,7 +19,6 @@ from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-m class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" - def setUp(self): self.cert = test_util.load_comparable_cert('rsa2048_cert.pem') key = test_util.load_pyopenssl_private_key('rsa2048_key.pem') @@ -34,7 +33,8 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init def server_bind(self): # pylint: disable=missing-docstring - self.socket = SSLSocket(socket.socket(), certs=certs) + self.socket = SSLSocket(socket.socket(), + certs) socketserver.TCPServer.server_bind(self) self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) @@ -66,6 +66,18 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # self.assertRaises(errors.Error, self._probe, b'bar') +class SSLSocketTest(unittest.TestCase): + """Tests for acme.crypto_util.SSLSocket.""" + + def test_ssl_socket_invalid_arguments(self): + from acme.crypto_util import SSLSocket + with self.assertRaises(ValueError): + _ = SSLSocket(None, {'sni': ('key', 'cert')}, + cert_selection=lambda _: None) + with self.assertRaises(ValueError): + _ = SSLSocket(None) + + class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index ff9159933..3bcb0b230 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -43,7 +43,14 @@ class TLSServer(socketserver.TCPServer): def _wrap_sock(self): self.socket = crypto_util.SSLSocket( - self.socket, certs=self.certs, method=self.method) + self.socket, cert_selection=self._cert_selection, + alpn_selection=getattr(self, '_alpn_selection', None), + method=self.method) + + def _cert_selection(self, connection): + """Callback selecting certificate for connection.""" + server_name = connection.get_servername() + return self.certs.get(server_name, None) def server_bind(self): # pylint: disable=missing-docstring self._wrap_sock() @@ -147,6 +154,45 @@ class TLSSNI01DualNetworkedServers(BaseDualNetworkedServers): BaseDualNetworkedServers.__init__(self, TLSSNI01Server, *args, **kwargs) +class BadALPNProtos(Exception): + """Error raised when cannot negotiate ALPN protocol.""" + pass + + +class TLSALPN01Server(TLSServer, ACMEServerMixin): + """TLSALPN01 Server.""" + + ACME_TLS_1_PROTOCOL = b"acme-tls/1" + + def __init__(self, server_address, certs, challenge_certs, ipv6=False): + TLSServer.__init__( + self, server_address, BaseRequestHandlerWithLogging, certs=certs, + ipv6=ipv6) + self.challenge_certs = challenge_certs + + def _cert_selection(self, connection): + # TODO: We would like to serve challenge cert only if asked for it via + # ALPN. To do this, we need to retrieve the list of protos from client + # hello, but this is currently impossible with openssl [0], and ALPN + # negotiation is done after cert selection. + # Therefore, currently we always return challenge cert, and terminate + # handshake in alpn_selection() if ALPN protos are not what we expect. + # [0] https://github.com/openssl/openssl/issues/4952 + server_name = connection.get_servername() + logger.debug("Serving challenge cert for server name %s", server_name) + return self.challenge_certs.get(server_name, None) + + def _alpn_selection(self, _connection, alpn_protos): + """Callback to select alpn protocol.""" + if len(alpn_protos) == 1 and alpn_protos[0] == self.ACME_TLS_1_PROTOCOL: + logger.debug("Agreed on %s ALPN", self.ACME_TLS_1_PROTOCOL) + return self.ACME_TLS_1_PROTOCOL + # Raising an exception causes openssl to terminate handshake and + # send fatal tls alert. + logger.debug("Cannot agree on ALPN proto. Got: %s", str(alpn_protos)) + raise BadALPNProtos("Got: %s" % str(alpn_protos)) + + class BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler): """BaseRequestHandler with logging.""" diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 1591187e5..aee592187 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -10,6 +10,7 @@ import unittest from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 import josepy as jose import mock import requests @@ -119,6 +120,62 @@ class HTTP01ServerTest(unittest.TestCase): self.assertFalse(self._test_http01(add=False)) +@unittest.skipUnless( + hasattr(SSL.Connection, "set_alpn_protos") and + hasattr(SSL.Context, "set_alpn_select_callback"), + "pyOpenSSL too old") +class TLSALPN01ServerTest(unittest.TestCase): + """Test for acme.standalone.TLSALPN01Server.""" + + def setUp(self): + self.certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa2048_key.pem'), + test_util.load_cert('rsa2048_cert.pem'), + )} + # Use different certificate for challenge. + self.challenge_certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa1024_key.pem'), + test_util.load_cert('rsa1024_cert.pem'), + )} + from acme.standalone import TLSALPN01Server + self.server = TLSALPN01Server(("", 0), certs=self.certs, + challenge_certs=self.challenge_certs) + # pylint: disable=no-member + self.thread = threading.Thread(target=self.server.serve_forever) + self.thread.start() + + def tearDown(self): + self.server.shutdown() # pylint: disable=no-member + self.thread.join() + + #TODO: This is not implemented yet, see comments in standalone.py + #def test_certs(self): + # host, port = self.server.socket.getsockname()[:2] + # cert = crypto_util.probe_sni( + # b'localhost', host=host, port=port, timeout=1) + # # Expect normal cert when connecting without ALPN. + # self.assertEqual(jose.ComparableX509(cert), + # jose.ComparableX509(self.certs[b'localhost'][1])) + + def test_challenge_certs(self): + host, port = self.server.socket.getsockname()[:2] + cert = crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"acme-tls/1"]) + # Expect challenge cert when connecting with ALPN. + self.assertEqual( + jose.ComparableX509(cert), + jose.ComparableX509(self.challenge_certs[b'localhost'][1]) + ) + + def test_bad_alpn(self): + host, port = self.server.socket.getsockname()[:2] + with self.assertRaises(errors.Error): + crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"bad-alpn"]) + + class BaseDualNetworkedServersTest(unittest.TestCase): """Test for acme.standalone.BaseDualNetworkedServers.""" diff --git a/acme/acme/testdata/README b/acme/acme/testdata/README index dfe3f5405..d65cc3018 100644 --- a/acme/acme/testdata/README +++ b/acme/acme/testdata/README @@ -10,6 +10,8 @@ and for the CSR: openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -outform DER > csr.der -and for the certificate: +and for the certificates: - openssl req -key rsa2047_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 > rsa2048_cert.pem + openssl req -key rsa1024_key.pem -new -subj '/CN=example.com' -x509 > rsa1024_cert.pem diff --git a/acme/acme/testdata/rsa1024_cert.pem b/acme/acme/testdata/rsa1024_cert.pem new file mode 100644 index 000000000..1b7912181 --- /dev/null +++ b/acme/acme/testdata/rsa1024_cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB/TCCAWagAwIBAgIJAOyRIBs3QT8QMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV +BAMMC2V4YW1wbGUuY29tMB4XDTE4MDQyMzEwMzE0NFoXDTE4MDUyMzEwMzE0NFow +FjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ +AoGBAJqJ87R8aVwByONxgQA9hwgvQd/QqI1r1UInXhEF2VnEtZGtUWLi100IpIqr +Mq4qusDwNZ3g8cUPtSkvJGs89djoajMDIJP7lQUEKUYnYrI0q755Tr/DgLWSk7iW +l5ezym0VzWUD0/xXUz8yRbNMTjTac80rS5SZk2ja2wWkYlRJAgMBAAGjUzBRMB0G +A1UdDgQWBBSsaX0IVZ4XXwdeffVAbG7gnxSYjTAfBgNVHSMEGDAWgBSsaX0IVZ4X +XwdeffVAbG7gnxSYjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB +ADe7SVmvGH2nkwVfONk8TauRUDkePN1CJZKFb2zW1uO9ANJ2v5Arm/OQp0BG/xnI +Djw/aLTNVESF89oe15dkrUErtcaF413MC1Ld5lTCaJLHLGqDKY69e02YwRuxW7jY +qarpt7k7aR5FbcfO5r4V/FK/Gvp4Dmoky8uap7SJIW6x +-----END CERTIFICATE----- From 236f9630e074ad45aa2b834483cbf373aae18566 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 4 Jun 2018 15:04:56 -0700 Subject: [PATCH 83/93] Remove unneeded sys import (#5873) * Remove unneeded sys import. Once upon a time we needed this in some of these setup.py files because we were using sys in the file, but we aren't anymore so let's remove the import. * use setuptools instead of distutils --- acme/setup.py | 2 -- certbot-apache/setup.py | 2 -- certbot-dns-cloudflare/setup.py | 2 -- certbot-dns-cloudxns/setup.py | 2 -- certbot-dns-digitalocean/setup.py | 2 -- certbot-dns-dnsimple/setup.py | 2 -- certbot-dns-dnsmadeeasy/setup.py | 2 -- certbot-dns-google/setup.py | 2 -- certbot-dns-luadns/setup.py | 2 -- certbot-dns-nsone/setup.py | 2 -- certbot-dns-rfc2136/setup.py | 2 -- certbot-dns-route53/setup.py | 4 +--- certbot-nginx/setup.py | 2 -- letshelp-certbot/setup.py | 2 -- setup.py | 1 - 15 files changed, 1 insertion(+), 30 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index e91c36b3d..f8196e953 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 0e4304300..59ca3ed4a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 8e1f9d28b..d80a1ae20 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 05998ee6a..3bb7457e3 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index cd3b0613e..1d7c34197 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 10ee710cd..9c678ec60 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a7f44b989..66f3edd55 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index c171e5014..64b332b04 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 2c0e35308..8df4f1735 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 821a40655..4425ace16 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 21d9dec29..6ced18b5d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 083cd15ae..9ee587c4e 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,6 +1,4 @@ -import sys - -from distutils.core import setup +from setuptools import setup from setuptools import find_packages version = '0.25.0.dev0' diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6889d06e4..067f280d4 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index b5be07a59..28ce0e962 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -1,5 +1,3 @@ -import sys - from setuptools import setup from setuptools import find_packages diff --git a/setup.py b/setup.py index ee0470d3a..e32925583 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ import codecs import os import re -import sys from setuptools import setup from setuptools import find_packages From 8e4303af9f30f514fce04b88bff70b6bbb9d43c8 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 4 Jun 2018 16:04:47 -0700 Subject: [PATCH 84/93] Reuse ACMEv1 accounts for ACMEv2 (#5902) * Reuse ACMEv1 accounts for ACMEv2 * Correct behavior * add unit tests * add _find_all_inner to comply with interface * acme-staging-v01 --> acme-staging * only create symlink to previous account if there is one there * recurse on server path * update tests and change internal use of load to use server_path * fail gracefully on corrupted account file by returning [] when rmdir fails * only reuse accounts in staging for now --- certbot/account.py | 42 ++++++++++++++++++++---- certbot/configuration.py | 6 +++- certbot/constants.py | 6 ++++ certbot/tests/account_test.py | 62 ++++++++++++++++++++++++++++++++++- 4 files changed, 108 insertions(+), 8 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index 70d9a7fc3..c4eeb1388 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -16,6 +16,7 @@ import zope.component from acme import fields as acme_fields from acme import messages +from certbot import constants from certbot import errors from certbot import interfaces from certbot import util @@ -142,7 +143,11 @@ class AccountFileStorage(interfaces.AccountStorage): self.config.strict_permissions) def _account_dir_path(self, account_id): - return os.path.join(self.config.accounts_dir, account_id) + return self._account_dir_path_for_server_path(account_id, self.config.server_path) + + def _account_dir_path_for_server_path(self, account_id, server_path): + accounts_dir = self.config.accounts_dir_for_server_path(server_path) + return os.path.join(accounts_dir, account_id) @classmethod def _regr_path(cls, account_dir_path): @@ -156,22 +161,44 @@ class AccountFileStorage(interfaces.AccountStorage): def _metadata_path(cls, account_dir_path): return os.path.join(account_dir_path, "meta.json") - def find_all(self): + def _find_all_for_server_path(self, server_path): + accounts_dir = self.config.accounts_dir_for_server_path(server_path) try: - candidates = os.listdir(self.config.accounts_dir) + candidates = os.listdir(accounts_dir) except OSError: return [] accounts = [] for account_id in candidates: try: - accounts.append(self.load(account_id)) + accounts.append(self._load_for_server_path(account_id, server_path)) except errors.AccountStorageError: logger.debug("Account loading problem", exc_info=True) + + + if not accounts and server_path in constants.LE_REUSE_SERVERS: + # find all for the next link down + prev_server_path = constants.LE_REUSE_SERVERS[server_path] + prev_accounts = self._find_all_for_server_path(prev_server_path) + # if we found something, link to that + if prev_accounts: + if os.path.islink(accounts_dir): + os.unlink(accounts_dir) + else: + try: + os.rmdir(accounts_dir) + except OSError: + return [] + prev_account_dir = self.config.accounts_dir_for_server_path(prev_server_path) + os.symlink(prev_account_dir, accounts_dir) + accounts = prev_accounts return accounts - def load(self, account_id): - account_dir_path = self._account_dir_path(account_id) + def find_all(self): + return self._find_all_for_server_path(self.config.server_path) + + def _load_for_server_path(self, account_id, server_path): + account_dir_path = self._account_dir_path_for_server_path(account_id, server_path) if not os.path.isdir(account_dir_path): raise errors.AccountNotFound( "Account at %s does not exist" % account_dir_path) @@ -193,6 +220,9 @@ class AccountFileStorage(interfaces.AccountStorage): account_id, acc.id)) return acc + def load(self, account_id): + return self._load_for_server_path(account_id, self.config.server_path) + def save(self, account, acme): self._save(account, acme, regr_only=False) diff --git a/certbot/configuration.py b/certbot/configuration.py index 297795609..daf514be8 100644 --- a/certbot/configuration.py +++ b/certbot/configuration.py @@ -65,8 +65,12 @@ class NamespaceConfig(object): @property def accounts_dir(self): # pylint: disable=missing-docstring + return self.accounts_dir_for_server_path(self.server_path) + + def accounts_dir_for_server_path(self, server_path): + """Path to accounts directory based on server_path""" return os.path.join( - self.namespace.config_dir, constants.ACCOUNTS_DIR, self.server_path) + self.namespace.config_dir, constants.ACCOUNTS_DIR, server_path) @property def backup_dir(self): # pylint: disable=missing-docstring diff --git a/certbot/constants.py b/certbot/constants.py index 1dd25e799..c6746e888 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -158,6 +158,12 @@ CONFIG_DIRS_MODE = 0o755 ACCOUNTS_DIR = "accounts" """Directory where all accounts are saved.""" +LE_REUSE_SERVERS = { + 'acme-staging-v02.api.letsencrypt.org/directory': + 'acme-staging.api.letsencrypt.org/directory' +} +"""Servers that can reuse accounts from other servers.""" + BACKUP_DIR = "backups" """Directory (relative to `IConfig.work_dir`) where backups are kept.""" diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 8ebda56af..a8059fbcf 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -95,6 +95,7 @@ class AccountMemoryStorageTest(unittest.TestCase): class AccountFileStorageTest(test_util.ConfigTestCase): """Tests for certbot.account.AccountFileStorage.""" + #pylint: disable=too-many-public-methods def setUp(self): super(AccountFileStorageTest, self).setUp() @@ -159,7 +160,8 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self.assertEqual([], self.storage.find_all()) def test_find_all_load_skips(self): - self.storage.load = mock.MagicMock( + # pylint: disable=protected-access + self.storage._load_for_server_path = mock.MagicMock( side_effect=["x", errors.AccountStorageError, "z"]) with mock.patch("certbot.account.os.listdir") as mock_listdir: mock_listdir.return_value = ["x", "y", "z"] @@ -175,6 +177,64 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self.assertRaises(errors.AccountStorageError, self.storage.load, "x" + self.acc.id) + def _set_server(self, server): + self.config.server = server + from certbot.account import AccountFileStorage + self.storage = AccountFileStorage(self.config) + + def test_find_all_neither_exists(self): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + self.assertEqual([], self.storage.find_all()) + self.assertFalse(os.path.islink(self.config.accounts_dir)) + + def test_find_all_find_before_save(self): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + self.storage.save(self.acc, self.mock_client) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertFalse(os.path.islink(self.config.accounts_dir)) + # we shouldn't have created a v1 account + prev_server_path = 'https://acme-staging.api.letsencrypt.org/directory' + self.assertFalse(os.path.isdir(self.config.accounts_dir_for_server_path(prev_server_path))) + + def test_find_all_save_before_find(self): + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertEqual([self.acc], self.storage.find_all()) + self.assertFalse(os.path.islink(self.config.accounts_dir)) + self.assertTrue(os.path.isdir(self.config.accounts_dir)) + prev_server_path = 'https://acme-staging.api.letsencrypt.org/directory' + self.assertFalse(os.path.isdir(self.config.accounts_dir_for_server_path(prev_server_path))) + + def test_find_all_server_downgrade(self): + # don't use v2 accounts with a v1 url + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + self.storage.save(self.acc, self.mock_client) + self.assertEqual([self.acc], self.storage.find_all()) + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + + def test_upgrade_version(self): + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([self.acc], self.storage.find_all()) + + @mock.patch('os.rmdir') + def test_corrupted_account(self, mock_rmdir): + # pylint: disable=protected-access + self._set_server('https://acme-staging.api.letsencrypt.org/directory') + self.storage.save(self.acc, self.mock_client) + mock_rmdir.side_effect = OSError + self.storage._load_for_server_path = mock.MagicMock( + side_effect=errors.AccountStorageError) + self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') + self.assertEqual([], self.storage.find_all()) + def test_load_ioerror(self): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() From 09a28c7a27fd01d1517e6381ffb37501bffca292 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Mon, 4 Jun 2018 17:44:51 -0700 Subject: [PATCH 85/93] Allow multiple add_headers directives (#6068) * fix(nginx-hsts): allow multiple add_headers * test(nginx): fix nginx tests --- certbot-nginx/certbot_nginx/parser.py | 2 +- .../certbot_nginx/tests/configurator_test.py | 15 ++++++++++++--- certbot-nginx/certbot_nginx/tests/parser_test.py | 5 +++-- .../testdata/etc_nginx/sites-enabled/headers.com | 4 ++++ 4 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 5bc7946dc..7d1da2e73 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -566,7 +566,7 @@ def _update_or_add_directives(directives, insert_at_top, block): INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite']) +REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite', 'add_header']) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index e88dcb8e0..0668a38c9 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -47,7 +47,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(10, len(self.config.parser.parsed)) + self.assertEqual(11, len(self.config.parser.parsed)) @mock.patch("certbot_nginx.configurator.util.exe_exists") @mock.patch("certbot_nginx.configurator.subprocess.Popen") @@ -91,7 +91,8 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(names, set( ["155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", - "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com"])) + "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", + "headers.com"])) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -548,6 +549,14 @@ class NginxConfiguratorTest(util.NginxTest): generated_conf = self.config.parser.parsed[example_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + def test_multiple_headers_hsts(self): + headers_conf = self.config.parser.abs_path('sites-enabled/headers.com') + self.config.enhance("headers.com", "ensure-http-header", + "Strict-Transport-Security") + expected = ['add_header', 'Strict-Transport-Security', '"max-age=31536000"', 'always'] + generated_conf = self.config.parser.parsed[headers_conf] + self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + def test_http_header_hsts_twice(self): self.config.enhance("www.example.com", "ensure-http-header", "Strict-Transport-Security") @@ -852,7 +861,7 @@ class NginxConfiguratorTest(util.NginxTest): prefer_ssl=False, no_ssl_filter_port='80') # Check that the dialog was called with only port 80 vhosts - self.assertEqual(len(mock_select_vhs.call_args[0][0]), 4) + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 5) class InstallSslOptionsConfTest(util.NginxTest): diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 5a37c9565..f6f28e42b 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -49,6 +49,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ['foo.conf', 'nginx.conf', 'server.conf', 'sites-enabled/default', 'sites-enabled/example.com', + 'sites-enabled/headers.com', 'sites-enabled/migration.com', 'sites-enabled/sslon.com', 'sites-enabled/globalssl.com', @@ -77,7 +78,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(7, len( + self.assertEqual(8, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -160,7 +161,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods '*.www.example.com']), [], [2, 1, 0]) - self.assertEqual(12, len(vhosts)) + self.assertEqual(13, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com new file mode 100644 index 000000000..6c032928c --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com @@ -0,0 +1,4 @@ +server { + server_name headers.com; + add_header X-Content-Type-Options nosniff; +} From d905886f4c63d1546ff723a909a088ab9334b91f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 5 Jun 2018 13:40:48 -0700 Subject: [PATCH 86/93] Automatically select among default vhosts if we have a port preference in nginx (#5944) * automatically select among default vhosts if we have a port preference * ports should be strings in the nginx plugin * clarify port vs preferred_port behavior by adding allow_port_mismatch flag * update all instances of default_vhosts to all_default_vhosts * require port * port should never be None in _get_default_vhost --- certbot-nginx/certbot_nginx/configurator.py | 31 ++++++++++++------- .../certbot_nginx/tests/configurator_test.py | 7 +++++ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 293f7378e..41b5124b8 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -289,7 +289,8 @@ class NginxConfigurator(common.Installer): if not vhosts: if create_if_no_match: # result will not be [None] because it errors on failure - vhosts = [self._vhost_from_duplicated_default(target_name)] + vhosts = [self._vhost_from_duplicated_default(target_name, True, + str(self.config.tls_sni_01_port))] else: # No matches. Raise a misconfiguration error. raise errors.MisconfigurationError( @@ -332,9 +333,12 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain, port=None): + def _vhost_from_duplicated_default(self, domain, allow_port_mismatch, port): + """if allow_port_mismatch is False, only server blocks with matching ports will be + used as a default server block template. + """ if self.new_vhost is None: - default_vhost = self._get_default_vhost(port, domain) + default_vhost = self._get_default_vhost(domain, allow_port_mismatch, port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, remove_singleton_listen_params=True) self.new_vhost.names = set() @@ -350,19 +354,24 @@ class NginxConfigurator(common.Installer): name_block[0].append(name) self.parser.update_or_add_server_directives(vhost, name_block) - def _get_default_vhost(self, port, domain): + def _get_default_vhost(self, domain, allow_port_mismatch, port): + """Helper method for _vhost_from_duplicated_default; see argument documentation there""" vhost_list = self.parser.get_vhosts() # if one has default_server set, return that one - default_vhosts = [] + all_default_vhosts = [] + port_matching_vhosts = [] for vhost in vhost_list: for addr in vhost.addrs: if addr.default: - if port is None or self._port_matches(port, addr.get_port()): - default_vhosts.append(vhost) - break + all_default_vhosts.append(vhost) + if self._port_matches(port, addr.get_port()): + port_matching_vhosts.append(vhost) + break - if len(default_vhosts) == 1: - return default_vhosts[0] + if len(port_matching_vhosts) == 1: + return port_matching_vhosts[0] + elif len(all_default_vhosts) == 1 and allow_port_mismatch: + return all_default_vhosts[0] # TODO: present a list of vhosts for user to choose from @@ -471,7 +480,7 @@ class NginxConfigurator(common.Installer): matches = self._get_redirect_ranked_matches(target_name, port) vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] if not vhosts and create_if_no_match: - vhosts = [self._vhost_from_duplicated_default(target_name, port=port)] + vhosts = [self._vhost_from_duplicated_default(target_name, False, port)] return vhosts def _port_matches(self, test_port, matching_port): diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 0668a38c9..9386c3cd9 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -731,6 +731,13 @@ class NginxConfiguratorTest(util.NginxTest): "www.nomatch.com", "example/cert.pem", "example/key.pem", "example/chain.pem", "example/fullchain.pem") + def test_deploy_no_match_multiple_defaults_ok(self): + foo_conf = self.config.parser.abs_path('foo.conf') + self.config.parser.parsed[foo_conf][2][1][0][1][0][1] = '*:5001' + self.config.version = (1, 3, 1) + self.config.deploy_cert("www.nomatch.com", "example/cert.pem", "example/key.pem", + "example/chain.pem", "example/fullchain.pem") + def test_deploy_no_match_add_redirect(self): default_conf = self.config.parser.abs_path('sites-enabled/default') foo_conf = self.config.parser.abs_path('foo.conf') From 868e5b831b27ac07cb4a73e04ed9e6fac5987fc7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 5 Jun 2018 17:59:11 -0700 Subject: [PATCH 87/93] Make python setup.py test use pytest for acme (#6072) --- acme/setup.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/acme/setup.py b/acme/setup.py index f8196e953..0fb4d8fff 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,6 +1,7 @@ from setuptools import setup from setuptools import find_packages - +from setuptools.command.test import test as TestCommand +import sys version = '0.25.0.dev0' @@ -33,6 +34,19 @@ docs_extras = [ 'sphinx_rtd_theme', ] +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) setup( name='acme', @@ -65,5 +79,7 @@ setup( 'dev': dev_extras, 'docs': docs_extras, }, + tests_require=["pytest"], test_suite='acme', + cmdclass={"test": PyTest}, ) From 3cffe1449c4e9166b65eaed75022d73b7ad79328 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jun 2018 07:58:50 -0700 Subject: [PATCH 88/93] Revert "switch signature verification to use pure cryptography (#6000)" (#6074) This reverts commit 366c50e28ee865f697f9e32e5b86e49dbf3ec5a2. --- certbot/crypto_util.py | 36 +++++-------------- certbot/tests/crypto_util_test.py | 10 ------ .../tests/testdata/cert-nosans_nistp256.pem | 11 ------ .../tests/testdata/csr-nosans_nistp256.pem | 8 ----- certbot/tests/testdata/nistp256_key.pem | 5 --- 5 files changed, 9 insertions(+), 61 deletions(-) delete mode 100644 certbot/tests/testdata/cert-nosans_nistp256.pem delete mode 100644 certbot/tests/testdata/csr-nosans_nistp256.pem delete mode 100644 certbot/tests/testdata/nistp256_key.pem diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 71f6c990c..b5ad16db1 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -12,16 +12,11 @@ import os import pyrfc3339 import six import zope.component -from cryptography.exceptions import InvalidSignature -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric.ec import ECDSA -from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey -from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 -from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey -# https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore +from cryptography.hazmat.backends import default_backend +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module @@ -233,26 +228,13 @@ def verify_renewable_cert_sig(renewable_cert): """ try: with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] - chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) + chain, _ = pyopenssl_load_certificate(chain_file.read()) with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] - cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) - pk = chain.public_key() - if isinstance(pk, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = pk.verifier( # type: ignore - cert.signature, PKCS1v15(), cert.signature_hash_algorithm - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - elif isinstance(pk, EllipticCurvePublicKey): - verifier = pk.verifier( - cert.signature, ECDSA(cert.signature_hash_algorithm) - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - else: - raise errors.Error("Unsupported public key type") - except (IOError, ValueError, InvalidSignature) as e: + cert = x509.load_pem_x509_certificate( + cert_file.read(), default_backend()) + hash_name = cert.signature_hash_algorithm.name + crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) + except (IOError, ValueError, crypto.Error) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index baf14b2ef..2fe0e3d30 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -21,9 +21,6 @@ CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.load_vector('cert_512.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') SS_CERT = test_util.load_vector('cert_2048.pem') -P256_KEY = test_util.load_vector('nistp256_key.pem') -P256_CERT_PATH = test_util.vector_path('cert-nosans_nistp256.pem') -P256_CERT = test_util.load_vector('cert-nosans_nistp256.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" @@ -220,13 +217,6 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): def test_cert_sig_match(self): self.assertEqual(None, self._call(self.renewable_cert)) - def test_cert_sig_match_ec(self): - renewable_cert = mock.MagicMock() - renewable_cert.cert = P256_CERT_PATH - renewable_cert.chain = P256_CERT_PATH - renewable_cert.privkey = P256_KEY - self.assertEqual(None, self._call(renewable_cert)) - def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert = test_util.vector_path('cert_512_bad.pem') self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) diff --git a/certbot/tests/testdata/cert-nosans_nistp256.pem b/certbot/tests/testdata/cert-nosans_nistp256.pem deleted file mode 100644 index 4ec3f24ce..000000000 --- a/certbot/tests/testdata/cert-nosans_nistp256.pem +++ /dev/null @@ -1,11 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBoDCCAUYCCQDCnzfUZ7TQdDAKBggqhkjOPQQDAjBYMQswCQYDVQQGEwJVUzER -MA8GA1UECAwITWljaGlnYW4xEjAQBgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwD -RUZGMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0xODA1MTUxNzIyMzlaFw0xODA2 -MTQxNzIyMzlaMFgxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhNaWNoaWdhbjESMBAG -A1UEBwwJQW5uIEFyYm9yMQwwCgYDVQQKDANFRkYxFDASBgNVBAMMC2V4YW1wbGUu -Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPPl0JauSZukvAUWv4l5VNLAY -QXhuPXYQBf4dVET3s0E5q9ZCbSe+pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tzAK -BggqhkjOPQQDAgNIADBFAiEAv8S2GXmWJqZ+j3DBfm72E1YK+HkOf+TOUHsbVR+O -Z1oCIFWNt1SPdIgRp4QAyzVk2pcTF8jDNajEMLWETDtxgRvM ------END CERTIFICATE----- diff --git a/certbot/tests/testdata/csr-nosans_nistp256.pem b/certbot/tests/testdata/csr-nosans_nistp256.pem deleted file mode 100644 index 2f0a671ed..000000000 --- a/certbot/tests/testdata/csr-nosans_nistp256.pem +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIBFDCBugIBADBYMQswCQYDVQQGEwJVUzERMA8GA1UECAwITWljaGlnYW4xEjAQ -BgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwDRUZGMRQwEgYDVQQDDAtleGFtcGxl -LmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDz5dCWrkmbpLwFFr+JeVTSw -GEF4bj12EAX+HVRE97NBOavWQm0nvqTVG5KPRfkxZLnO11Y0D7H5A24dCPZw9Leg -ADAKBggqhkjOPQQDAgNJADBGAiEAuoZHrYA5sy2DRTdLAxJTBNHKFFKbtaGt+QaJ -A62qa8sCIQCUkSgSAiNaEnJ7r5fKphdjeORHqhpl6flYkLE3lGmGdg== ------END CERTIFICATE REQUEST----- diff --git a/certbot/tests/testdata/nistp256_key.pem b/certbot/tests/testdata/nistp256_key.pem deleted file mode 100644 index 4be37e49b..000000000 --- a/certbot/tests/testdata/nistp256_key.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIOvXH384CyNNv2lfxvjc7hg2f7ScYoLvlk/VpINLJlGBoAoGCCqGSM49 -AwEHoUQDQgAEPPl0JauSZukvAUWv4l5VNLAYQXhuPXYQBf4dVET3s0E5q9ZCbSe+ -pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tw== ------END EC PRIVATE KEY----- From eec37f65a83ccf185a67378b6b8010b37f4fba1f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jun 2018 19:01:55 -0700 Subject: [PATCH 89/93] Update changelog for 0.25.0 (#6076) --- CHANGELOG.md | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 044e55250..5facc5380 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,70 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.25.0 - 2018-06-06 + +### Added + +* Support for the ready status type was added to acme. Without this change, + Certbot and acme users will begin encountering errors when using Let's + Encrypt's ACMEv2 API starting on June 19th for the staging environment and + July 5th for production. See + https://community.letsencrypt.org/t/acmev2-order-ready-status/62866 for more + information. +* Certbot now accepts the flag --reuse-key which will cause the same key to be + used in the certificate when the lineage is renewed rather than generating a + new key. +* You can now add multiple email addresses to your ACME account with Certbot by + providing a comma separated list of emails to the --email flag. +* Support for Let's Encrypt's upcoming TLS-ALPN-01 challenge was added to acme. + For more information, see + https://community.letsencrypt.org/t/tls-alpn-validation-method/63814/1. +* acme now supports specifying the source address to bind to when sending + outgoing connections. You still cannot specify this address using Certbot. +* If you run Certbot against Let's Encrypt's ACMEv2 staging server but don't + already have an account registered at that server URL, Certbot will + automatically reuse your staging account from Let's Encrypt's ACMEv1 endpoint + if it exists. +* Interfaces were added to Certbot allowing plugins to be called at additional + points. The `GenericUpdater` interface allows plugins to perform actions + every time `certbot renew` is run, regardless of whether any certificates are + due for renewal, and the `RenewDeployer` interface allows plugins to perform + actions when a certificate is renewed. See `certbot.interfaces` for more + information. + +### Changed + +* When running Certbot with --dry-run and you don't already have a staging + account, the created account does not contain an email address even if one + was provided to avoid expiration emails from Let's Encrypt's staging server. +* certbot-nginx does a better job of automatically detecting the location of + Nginx's configuration files when run on BSD based systems. +* acme now requires and uses pytest when running tests with setuptools with + `python setup.py test`. +* `certbot config_changes` no longer waits for user input before exiting. + +### Fixed + +* Misleading log output that caused users to think that Certbot's standalone + plugin failed to bind to a port when performing a challenge has been + corrected. +* An issue where certbot-nginx would fail to enable HSTS if the server block + already had an `add_header` directive has been resolved. +* certbot-nginx now does a better job detecting the server block to base the + configuration for TLS-SNI challenges on. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +packages with functional changes were: + +* acme +* certbot +* certbot-apache +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/54?closed=1 + ## 0.24.0 - 2018-05-02 ### Added From da6320f4d1b816592991f144caa4d6197faafd14 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 01:11:06 -0700 Subject: [PATCH 90/93] Stop testing against Debian 7. (#6077) Debian Wheezy is no longer supported (see https://wiki.debian.org/LTS) and Amazon shut down their Debian 7 mirrors so let's stop trying to use Debian 7 during testing. --- tests/letstest/targets.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 9c1aca24e..766b4ea09 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -22,24 +22,6 @@ targets: # #cloud-init # runcmd: # - [ apt-get, install, -y, curl ] - - ami: ami-e0efab88 - name: debian7.8.aws.1 - type: ubuntu - virt: hvm - user: admin - # userdata: | - # #cloud-init - # runcmd: - # - [ apt-get, install, -y, curl ] - - ami: ami-e6eeaa8e - name: debian7.8.aws.1_32bit - type: ubuntu - virt: pv - user: admin - # userdata: | - # #cloud-init - # runcmd: - # - [ apt-get, install, -y, curl ] #----------------------------------------------------------------------------- # Other Redhat Distros - ami: ami-60b6c60a From 780a1b3a26192f06c58fce157db5c2a32ecbdc23 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 01:43:45 -0700 Subject: [PATCH 91/93] Don't require festival during signing. (#6079) Festival isn't available via Homebrew and is only needed to read the hash aloud, so let's not make it a strict requirement that it's installed. You can simply read the hash from the terminal instead. --- tools/offline-sigrequest.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/offline-sigrequest.sh b/tools/offline-sigrequest.sh index 7706796ef..6443ae8af 100755 --- a/tools/offline-sigrequest.sh +++ b/tools/offline-sigrequest.sh @@ -2,17 +2,17 @@ set -o errexit -if ! `which festival > /dev/null` ; then - echo Please install \'festival\'! - exit 1 -fi - function sayhash { # $1 <-- HASH ; $2 <---SIGFILEBALL while read -p "Press Enter to read the hash aloud or type 'done': " INP && [ "$INP" = "" ] ; do - cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \ - echo -n '(SayText "'; \ - sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ - echo '")' ) | festival + if ! `which festival > /dev/null` ; then + echo \`festival\` is not installed! + echo Please install it to read the hash aloud + else + cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \ + echo -n '(SayText "'; \ + sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ + echo '")' ) | festival + fi done echo 'Paste in the data from the QR code, then type Ctrl-D:' From 3a8de6d172d3d91348cb4b817586869a756cc2a8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 07:50:36 -0700 Subject: [PATCH 92/93] Upgrade pinned twine version. (#6078) For the past couple of releases, twine has errored while trying to upload packages and this is fixed by upgrading to a newer version of twine. This commit updates our pinned version installed when using tools/venv.sh to the latest available version. pkginfo had to be upgraded as well to support the latest version of twine. --- tools/dev_constraints.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index d965d4470..777222ffb 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -36,7 +36,7 @@ oauth2client==2.0.0 pathlib2==2.3.0 pexpect==4.2.1 pickleshare==0.7.4 -pkginfo==1.4.1 +pkginfo==1.4.2 pluggy==0.5.2 prompt-toolkit==1.0.15 ptyprocess==0.5.2 @@ -66,7 +66,7 @@ tldextract==2.2.0 tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 -twine==1.9.1 +twine==1.11.0 typed-ast==1.1.0 typing==3.6.4 uritemplate==0.6 From afa7e3fb8266fb6af7c36d46eb8ee20e844d8107 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jun 2018 14:45:23 -0700 Subject: [PATCH 93/93] Unrevert #6000 and silence deprecation warnings (#6082) * Revert "Revert "switch signature verification to use pure cryptography (#6000)" (#6074)" This reverts commit 3cffe1449c4e9166b65eaed75022d73b7ad79328. * Fixes #6073. This silences the deprecation warnings from cryptography. I looked into only silencing the cryptography warning specifically in the function, however, CryptographyDeprecationWarning doesn't seem to be publicly documented, so we probably shouldn't depend on it. --- certbot/crypto_util.py | 40 ++++++++++++++----- certbot/tests/crypto_util_test.py | 10 +++++ .../tests/testdata/cert-nosans_nistp256.pem | 11 +++++ .../tests/testdata/csr-nosans_nistp256.pem | 8 ++++ certbot/tests/testdata/nistp256_key.pem | 5 +++ 5 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 certbot/tests/testdata/cert-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/csr-nosans_nistp256.pem create mode 100644 certbot/tests/testdata/nistp256_key.pem diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index b5ad16db1..943e2c87f 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -7,16 +7,21 @@ import hashlib import logging import os - +import warnings import pyrfc3339 import six import zope.component +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +# https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore -from cryptography.hazmat.backends import default_backend -# https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module @@ -228,13 +233,28 @@ def verify_renewable_cert_sig(renewable_cert): """ try: with open(renewable_cert.chain, 'rb') as chain_file: # type: IO[bytes] - chain, _ = pyopenssl_load_certificate(chain_file.read()) + chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) with open(renewable_cert.cert, 'rb') as cert_file: # type: IO[bytes] - cert = x509.load_pem_x509_certificate( - cert_file.read(), default_backend()) - hash_name = cert.signature_hash_algorithm.name - crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name) - except (IOError, ValueError, crypto.Error) as e: + cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) + pk = chain.public_key() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + if isinstance(pk, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = pk.verifier( # type: ignore + cert.signature, PKCS1v15(), cert.signature_hash_algorithm + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + elif isinstance(pk, EllipticCurvePublicKey): + verifier = pk.verifier( + cert.signature, ECDSA(cert.signature_hash_algorithm) + ) + verifier.update(cert.tbs_certificate_bytes) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") + except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) logger.exception(error_str) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 2fe0e3d30..baf14b2ef 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -21,6 +21,9 @@ CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.load_vector('cert_512.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') SS_CERT = test_util.load_vector('cert_2048.pem') +P256_KEY = test_util.load_vector('nistp256_key.pem') +P256_CERT_PATH = test_util.vector_path('cert-nosans_nistp256.pem') +P256_CERT = test_util.load_vector('cert-nosans_nistp256.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" @@ -217,6 +220,13 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): def test_cert_sig_match(self): self.assertEqual(None, self._call(self.renewable_cert)) + def test_cert_sig_match_ec(self): + renewable_cert = mock.MagicMock() + renewable_cert.cert = P256_CERT_PATH + renewable_cert.chain = P256_CERT_PATH + renewable_cert.privkey = P256_KEY + self.assertEqual(None, self._call(renewable_cert)) + def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert = test_util.vector_path('cert_512_bad.pem') self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) diff --git a/certbot/tests/testdata/cert-nosans_nistp256.pem b/certbot/tests/testdata/cert-nosans_nistp256.pem new file mode 100644 index 000000000..4ec3f24ce --- /dev/null +++ b/certbot/tests/testdata/cert-nosans_nistp256.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBoDCCAUYCCQDCnzfUZ7TQdDAKBggqhkjOPQQDAjBYMQswCQYDVQQGEwJVUzER +MA8GA1UECAwITWljaGlnYW4xEjAQBgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwD +RUZGMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0xODA1MTUxNzIyMzlaFw0xODA2 +MTQxNzIyMzlaMFgxCzAJBgNVBAYTAlVTMREwDwYDVQQIDAhNaWNoaWdhbjESMBAG +A1UEBwwJQW5uIEFyYm9yMQwwCgYDVQQKDANFRkYxFDASBgNVBAMMC2V4YW1wbGUu +Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPPl0JauSZukvAUWv4l5VNLAY +QXhuPXYQBf4dVET3s0E5q9ZCbSe+pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tzAK +BggqhkjOPQQDAgNIADBFAiEAv8S2GXmWJqZ+j3DBfm72E1YK+HkOf+TOUHsbVR+O +Z1oCIFWNt1SPdIgRp4QAyzVk2pcTF8jDNajEMLWETDtxgRvM +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/csr-nosans_nistp256.pem b/certbot/tests/testdata/csr-nosans_nistp256.pem new file mode 100644 index 000000000..2f0a671ed --- /dev/null +++ b/certbot/tests/testdata/csr-nosans_nistp256.pem @@ -0,0 +1,8 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBFDCBugIBADBYMQswCQYDVQQGEwJVUzERMA8GA1UECAwITWljaGlnYW4xEjAQ +BgNVBAcMCUFubiBBcmJvcjEMMAoGA1UECgwDRUZGMRQwEgYDVQQDDAtleGFtcGxl +LmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDz5dCWrkmbpLwFFr+JeVTSw +GEF4bj12EAX+HVRE97NBOavWQm0nvqTVG5KPRfkxZLnO11Y0D7H5A24dCPZw9Leg +ADAKBggqhkjOPQQDAgNJADBGAiEAuoZHrYA5sy2DRTdLAxJTBNHKFFKbtaGt+QaJ +A62qa8sCIQCUkSgSAiNaEnJ7r5fKphdjeORHqhpl6flYkLE3lGmGdg== +-----END CERTIFICATE REQUEST----- diff --git a/certbot/tests/testdata/nistp256_key.pem b/certbot/tests/testdata/nistp256_key.pem new file mode 100644 index 000000000..4be37e49b --- /dev/null +++ b/certbot/tests/testdata/nistp256_key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIOvXH384CyNNv2lfxvjc7hg2f7ScYoLvlk/VpINLJlGBoAoGCCqGSM49 +AwEHoUQDQgAEPPl0JauSZukvAUWv4l5VNLAYQXhuPXYQBf4dVET3s0E5q9ZCbSe+ +pNUbko9F+TFkuc7XVjQPsfkDbh0I9nD0tw== +-----END EC PRIVATE KEY-----