From 2bc64183a8fe963959e91bfd87fb8f0e64b17650 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 11 Nov 2019 17:11:47 -0800 Subject: [PATCH 01/92] fix docstring --- certbot/plugins/common_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 977500f86..5d529e993 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -178,7 +178,7 @@ class InstallerTest(test_util.ConfigTestCase): class AddrTest(unittest.TestCase): - """Tests for certbot._internal.client.plugins.common.Addr.""" + """Tests for certbot._internal.plugins.common.Addr.""" def setUp(self): from certbot.plugins.common import Addr From 86926dff9293ab24d4f025ccf70c47619750d897 Mon Sep 17 00:00:00 2001 From: OsirisInferi Date: Tue, 4 Feb 2020 19:27:27 +0100 Subject: [PATCH 02/92] Use unrestrictive umask for challenge directory --- certbot-apache/certbot_apache/_internal/http_01.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index c34abc2b4..53ccd2bc7 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -168,7 +168,9 @@ class ApacheHttp01(common.ChallengePerformer): def _set_up_challenges(self): if not os.path.isdir(self.challenge_dir): + old_umask = os.umask(0o022) filesystem.makedirs(self.challenge_dir, 0o755) + os.umask(old_umask) responses = [] for achall in self.achalls: From 601a114d1ba6030f3f765ff86bb39658172e0a75 Mon Sep 17 00:00:00 2001 From: OsirisInferi Date: Tue, 4 Feb 2020 19:47:27 +0100 Subject: [PATCH 03/92] Update changelog --- certbot/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 86d27143c..01cd3d402 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -19,6 +19,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed * Fix collections.abc imports for Python 3.9. +* Fix Apache plugin to use less restrictive umask for making the challenge directory when a restrictive umask was set when certbot was started. More details about these changes can be found on our GitHub repo. From f3ed13374456f3b53fc87dc0fa1ed71b1efa37e7 Mon Sep 17 00:00:00 2001 From: OsirisInferi Date: Wed, 5 Feb 2020 22:17:29 +0100 Subject: [PATCH 04/92] Wrap makedirs() within exception handelrs --- certbot-apache/certbot_apache/_internal/http_01.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 53ccd2bc7..ad62a77bb 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -169,8 +169,14 @@ class ApacheHttp01(common.ChallengePerformer): def _set_up_challenges(self): if not os.path.isdir(self.challenge_dir): old_umask = os.umask(0o022) - filesystem.makedirs(self.challenge_dir, 0o755) - os.umask(old_umask) + try: + filesystem.makedirs(self.challenge_dir, 0o755) + except OSError as exception: + if exception.errno not in (errno.EEXIST, errno.EISDIR): + raise errors.PluginError( + "Couldn't create root for http-01 challenge") + finally: + os.umask(old_umask) responses = [] for achall in self.achalls: From d3a4b8fd8c068624b40179f567e191b6979bf6cf Mon Sep 17 00:00:00 2001 From: OsirisInferi Date: Wed, 5 Feb 2020 22:27:12 +0100 Subject: [PATCH 05/92] Missing import --- certbot-apache/certbot_apache/_internal/http_01.py | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index ad62a77bb..6c822cc38 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -1,5 +1,6 @@ """A class that performs HTTP-01 challenges for Apache""" import logging +import errno from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module From 605ef40656110e4b1e9f5d35c47b102ea2aa2b4b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 11 Feb 2020 14:20:29 -0800 Subject: [PATCH 06/92] Remove duplicate pyparsing pin --- tools/oldest_constraints.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 6154b497a..85d058796 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -13,7 +13,6 @@ ply==3.4 pyasn1==0.1.9 pycparser==2.14 pyOpenSSL==0.13.1 -pyparsing==1.5.6 pyRFC3339==1.0 python-augeas==0.5.0 oauth2client==4.0.0 From 7d540fc33ad39ec6664807c1a6445426c5534097 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 11 Feb 2020 14:44:37 -0800 Subject: [PATCH 07/92] update pyparsing comment --- certbot-nginx/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 3b75a3424..b180fe06a 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -13,7 +13,7 @@ install_requires = [ 'certbot>=1.1.0', 'mock', 'PyOpenSSL', - 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? + 'pyparsing>=1.5.5', # Python3 support 'setuptools', 'zope.interface', ] From df584a3b90f9c18efaac07a5d7fb0bf9be5a81a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 12 Feb 2020 13:12:03 -0800 Subject: [PATCH 08/92] Remove _internal from docstring. --- certbot/tests/plugins/common_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index fc05bf894..7543f28f3 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -177,7 +177,7 @@ class InstallerTest(test_util.ConfigTestCase): class AddrTest(unittest.TestCase): - """Tests for certbot._internal.plugins.common.Addr.""" + """Tests for certbot.plugins.common.Addr.""" def setUp(self): from certbot.plugins.common import Addr From fc7e5e8e6060d9e0df2e704a20103d5c0f456925 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 13 Feb 2020 22:56:16 +0100 Subject: [PATCH 09/92] Remove useless pylint error suppression directives (#7657) As pylint is evolving, it improves its accuracy, and several pylint error suppression (`# pylint: disable=ERROR) added in certbot codebase months or years ago are not needed anymore to make it happy. There is a (disabled by default) pylint error to detect the useless suppressions (pylint-ception: `useless-suppression`). It is not working perfectly (it has also false-positives ...) but it is a good start to clean the codebase. This PR removes several of these useless suppressions as detected by the current pylint version we use. * Remove useless suppress * Remove useless lines --- acme/acme/challenges.py | 2 +- acme/acme/client.py | 14 ++++++------ acme/acme/crypto_util.py | 12 +++++----- acme/acme/jws.py | 4 ++-- acme/acme/magic_typing.py | 1 - acme/acme/messages.py | 7 +++--- acme/acme/standalone.py | 16 +++++--------- .../certbot_apache/_internal/configurator.py | 20 ++++++++--------- .../certbot_apache/_internal/entrypoint.py | 2 +- .../certbot_apache/_internal/http_01.py | 4 ++-- .../certbot_apache/_internal/obj.py | 2 +- .../_internal/override_centos.py | 2 +- .../certbot_apache/_internal/parser.py | 10 ++++----- .../configurators/nginx/common.py | 2 +- .../certbot_compatibility_test/test_driver.py | 4 ++-- .../certbot_compatibility_test/validator.py | 3 +-- .../_internal/dns_cloudflare.py | 2 +- .../_internal/dns_cloudxns.py | 2 +- .../_internal/dns_digitalocean.py | 2 +- .../_internal/dns_dnsimple.py | 2 +- .../_internal/dns_dnsmadeeasy.py | 2 +- .../_internal/dns_gehirn.py | 2 +- .../_internal/dns_google.py | 8 +++---- .../_internal/dns_linode.py | 2 +- .../_internal/dns_luadns.py | 2 +- .../certbot_dns_nsone/_internal/dns_nsone.py | 2 +- .../certbot_dns_ovh/_internal/dns_ovh.py | 2 +- .../_internal/dns_rfc2136.py | 2 +- .../_internal/dns_route53.py | 10 ++++----- .../_internal/dns_sakuracloud.py | 2 +- .../certbot_nginx/_internal/configurator.py | 14 ++++++------ .../certbot_nginx/_internal/http_01.py | 2 +- .../certbot_nginx/_internal/nginxparser.py | 1 - .../certbot_nginx/_internal/parser.py | 11 +++++----- .../certbot_nginx/_internal/parser_obj.py | 2 +- certbot/certbot/_internal/auth_handler.py | 6 ++--- certbot/certbot/_internal/cert_manager.py | 2 +- certbot/certbot/_internal/cli.py | 8 +++---- certbot/certbot/_internal/client.py | 6 ++--- certbot/certbot/_internal/configuration.py | 18 +++++++-------- certbot/certbot/_internal/error_handler.py | 10 ++++----- certbot/certbot/_internal/hooks.py | 4 ++-- certbot/certbot/_internal/lock.py | 10 ++++----- certbot/certbot/_internal/main.py | 2 +- certbot/certbot/_internal/plugins/disco.py | 5 +---- certbot/certbot/_internal/plugins/manual.py | 12 +++++----- certbot/certbot/_internal/plugins/null.py | 2 +- .../certbot/_internal/plugins/standalone.py | 22 +++++++++---------- certbot/certbot/_internal/plugins/webroot.py | 20 ++++++++--------- certbot/certbot/_internal/renewal.py | 2 +- certbot/certbot/_internal/reporter.py | 2 +- certbot/certbot/_internal/storage.py | 2 +- certbot/certbot/compat/_path.py | 2 +- certbot/certbot/compat/filesystem.py | 7 +++--- certbot/certbot/compat/misc.py | 2 +- certbot/certbot/crypto_util.py | 2 +- certbot/certbot/display/util.py | 1 - certbot/certbot/interfaces.py | 2 +- certbot/certbot/ocsp.py | 6 ++--- certbot/certbot/plugins/common.py | 3 +-- certbot/certbot/plugins/dns_common.py | 8 +++---- certbot/certbot/plugins/dns_common_lexicon.py | 6 ++--- certbot/certbot/plugins/enhancements.py | 6 ++--- certbot/certbot/plugins/storage.py | 4 ++-- certbot/certbot/reverter.py | 6 ++--- certbot/certbot/tests/acme_util.py | 3 +-- certbot/certbot/tests/util.py | 2 +- certbot/certbot/util.py | 8 +++---- letshelp-certbot/letshelp_certbot/apache.py | 2 +- .../letshelp_certbot/magic_typing.py | 1 - .../letshelp_certbot/magic_typing_test.py | 4 ++-- 71 files changed, 183 insertions(+), 202 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 39c8d6269..f3fb19b42 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -54,7 +54,7 @@ class UnrecognizedChallenge(Challenge): object.__setattr__(self, "jobj", jobj) def to_partial_json(self): - return self.jobj # pylint: disable=no-member + return self.jobj @classmethod def from_json(cls, jobj): diff --git a/acme/acme/client.py b/acme/acme/client.py index f48ff40b2..3e03748b5 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -15,16 +15,16 @@ import requests from requests.adapters import HTTPAdapter from requests_toolbelt.adapters.source import SourceAddressAdapter import six -from six.moves import http_client # pylint: disable=import-error +from six.moves import http_client from acme import crypto_util from acme import errors from acme import jws from acme import messages -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Text # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict +from acme.magic_typing import List +from acme.magic_typing import Set +from acme.magic_typing import Text logger = logging.getLogger(__name__) @@ -36,7 +36,7 @@ if sys.version_info < (2, 7, 9): # pragma: no cover try: requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() # type: ignore except AttributeError: - import urllib3.contrib.pyopenssl # pylint: disable=import-error + import urllib3.contrib.pyopenssl urllib3.contrib.pyopenssl.inject_into_urllib3() DEFAULT_NETWORK_TIMEOUT = 45 @@ -666,7 +666,7 @@ class ClientV2(ClientBase): response = self._post(self.directory['newOrder'], order) body = messages.Order.from_json(response.json()) authorizations = [] - for url in body.authorizations: # pylint: disable=not-an-iterable + for url in body.authorizations: authorizations.append(self._authzr_from_response(self._post_as_get(url), uri=url)) return messages.OrderResource( body=body, diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 66dfc738c..dc8fedad0 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -11,10 +11,9 @@ from OpenSSL import crypto from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from acme import errors -from acme.magic_typing import Callable # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Callable +from acme.magic_typing import Tuple +from acme.magic_typing import Union logger = logging.getLogger(__name__) @@ -74,7 +73,7 @@ class SSLSocket(object): class FakeConnection(object): """Fake OpenSSL.SSL.Connection.""" - # pylint: disable=missing-docstring + # pylint: disable=missing-function-docstring def __init__(self, connection): self._wrapped = connection @@ -86,7 +85,7 @@ class SSLSocket(object): # OpenSSL.SSL.Connection.shutdown doesn't accept any args return self._wrapped.shutdown() - def accept(self): # pylint: disable=missing-docstring + def accept(self): # pylint: disable=missing-function-docstring sock, addr = self.sock.accept() context = SSL.Context(self.method) @@ -298,7 +297,6 @@ def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM): def _dump_cert(cert): if isinstance(cert, jose.ComparableX509): - # pylint: disable=protected-access cert = cert.wrapped return crypto.dump_certificate(filetype, cert) diff --git a/acme/acme/jws.py b/acme/acme/jws.py index 894e69f3d..9128f56b3 100644 --- a/acme/acme/jws.py +++ b/acme/acme/jws.py @@ -15,7 +15,7 @@ class Header(jose.Header): url = jose.Field('url', omitempty=True) @nonce.decoder - def nonce(value): # pylint: disable=missing-docstring,no-self-argument + def nonce(value): # pylint: disable=no-self-argument,missing-function-docstring try: return jose.decode_b64jose(value) except jose.DeserializationError as error: @@ -25,7 +25,7 @@ class Header(jose.Header): class Signature(jose.Signature): """ACME-specific Signature. Uses ACME-specific Header for customer fields.""" - __slots__ = jose.Signature._orig_slots # pylint: disable=no-member + __slots__ = jose.Signature._orig_slots # TODO: decoder/encoder should accept cls? Otherwise, subclassing # JSONObjectWithFields is tricky... diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 5a6358c69..d6b1ff056 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -10,7 +10,6 @@ class TypingClass(object): try: # mypy doesn't respect modifying sys.modules from typing import * # pylint: disable=wildcard-import, unused-wildcard-import - # pylint: disable=unused-import from typing import Collection, IO # type: ignore # pylint: enable=unused-import except ImportError: diff --git a/acme/acme/messages.py b/acme/acme/messages.py index e82d12890..f8f4bfbe7 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -11,7 +11,7 @@ from acme import jws from acme import util try: - from collections.abc import Hashable # pylint: disable=no-name-in-module + from collections.abc import Hashable except ImportError: # pragma: no cover from collections import Hashable @@ -460,7 +460,6 @@ class ChallengeResource(Resource): @property def uri(self): """The URL of the challenge body.""" - # pylint: disable=function-redefined,no-member return self.body.uri @@ -488,7 +487,7 @@ class Authorization(ResourceBody): wildcard = jose.Field('wildcard', omitempty=True) @challenges.decoder - def challenges(value): # pylint: disable=missing-docstring,no-self-argument + def challenges(value): # pylint: disable=no-self-argument,missing-function-docstring return tuple(ChallengeBody.from_json(chall) for chall in value) @property @@ -585,7 +584,7 @@ class Order(ResourceBody): error = jose.Field('error', omitempty=True, decoder=Error.from_json) @identifiers.decoder - def identifiers(value): # pylint: disable=missing-docstring,no-self-argument + def identifiers(value): # pylint: disable=no-self-argument,missing-function-docstring return tuple(Identifier.from_json(identifier) for identifier in value) class OrderResource(ResourceWithURI): diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index cf0da4e86..236f2c234 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -5,19 +5,16 @@ import logging import socket import threading -from six.moves import BaseHTTPServer # type: ignore # pylint: disable=import-error -from six.moves import http_client # pylint: disable=import-error -from six.moves import socketserver # type: ignore # pylint: disable=import-error +from six.moves import BaseHTTPServer # type: ignore +from six.moves import http_client +from six.moves import socketserver # type: ignore from acme import challenges from acme import crypto_util -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List logger = logging.getLogger(__name__) -# six.moves.* | pylint: disable=no-member,attribute-defined-outside-init -# pylint: disable=no-init - class TLSServer(socketserver.TCPServer): """Generic TLS Server.""" @@ -30,7 +27,6 @@ class TLSServer(socketserver.TCPServer): self.address_family = socket.AF_INET self.certs = kwargs.pop("certs", {}) self.method = kwargs.pop( - # pylint: disable=protected-access "method", crypto_util._DEFAULT_SSL_METHOD) self.allow_reuse_address = kwargs.pop("allow_reuse_address", True) socketserver.TCPServer.__init__(self, *args, **kwargs) @@ -39,7 +35,7 @@ class TLSServer(socketserver.TCPServer): self.socket = crypto_util.SSLSocket( self.socket, certs=self.certs, method=self.method) - def server_bind(self): # pylint: disable=missing-docstring + def server_bind(self): self._wrap_sock() return socketserver.TCPServer.server_bind(self) @@ -178,7 +174,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.log_message("Incoming request") BaseHTTPServer.BaseHTTPRequestHandler.handle(self) - def do_GET(self): # pylint: disable=invalid-name,missing-docstring + def do_GET(self): # pylint: disable=invalid-name,missing-function-docstring if self.path == "/": self.handle_index() elif self.path.startswith("/" + challenges.HTTP01.URI_ROOT_PATH): diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index e9ed1f8ab..465237590 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -14,11 +14,11 @@ import zope.component import zope.interface from acme import challenges -from acme.magic_typing import DefaultDict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import DefaultDict +from acme.magic_typing import Dict +from acme.magic_typing import List +from acme.magic_typing import Set +from acme.magic_typing import Union from certbot import errors from certbot import interfaces from certbot import util @@ -792,7 +792,7 @@ class ApacheConfigurator(common.Installer): return util.get_filtered_names(all_names) - def get_name_from_ip(self, addr): # pylint: disable=no-self-use + def get_name_from_ip(self, addr): """Returns a reverse dns name if available. :param addr: IP Address @@ -1726,7 +1726,7 @@ class ApacheConfigurator(common.Installer): ###################################################################### # Enhancements ###################################################################### - def supported_enhancements(self): # pylint: disable=no-self-use + def supported_enhancements(self): """Returns currently supported enhancements.""" return ["redirect", "ensure-http-header", "staple-ocsp"] @@ -2292,7 +2292,7 @@ class ApacheConfigurator(common.Installer): vhost.enabled = True return - def enable_mod(self, mod_name, temp=False): # pylint: disable=unused-argument + def enable_mod(self, mod_name, temp=False): """Enables module in Apache. Both enables and reloads Apache so module is active. @@ -2350,7 +2350,7 @@ class ApacheConfigurator(common.Installer): error = str(err) raise errors.MisconfigurationError(error) - def config_test(self): # pylint: disable=no-self-use + def config_test(self): """Check the configuration of Apache for errors. :raises .errors.MisconfigurationError: If config_test fails @@ -2400,7 +2400,7 @@ class ApacheConfigurator(common.Installer): ########################################################################### # Challenges Section ########################################################################### - def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use + def get_chall_pref(self, unused_domain): """Return list of challenge preferences.""" return [challenges.HTTP01] diff --git a/certbot-apache/certbot_apache/_internal/entrypoint.py b/certbot-apache/certbot_apache/_internal/entrypoint.py index d43094976..e31e1f4eb 100644 --- a/certbot-apache/certbot_apache/_internal/entrypoint.py +++ b/certbot-apache/certbot_apache/_internal/entrypoint.py @@ -1,7 +1,7 @@ """ Entry point for Apache Plugin """ # Pylint does not like disutils.version when running inside a venv. # See: https://github.com/PyCQA/pylint/issues/73 -from distutils.version import LooseVersion # pylint: disable=no-name-in-module,import-error +from distutils.version import LooseVersion from certbot import util from certbot_apache._internal import configurator diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index c34abc2b4..5ea0ce8ec 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -1,8 +1,8 @@ """A class that performs HTTP-01 challenges for Apache""" import logging -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List +from acme.magic_typing import Set from certbot import errors from certbot.compat import filesystem from certbot.compat import os diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 940bb6144..498766744 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -1,7 +1,7 @@ """Module contains classes used by the Apache Configurator.""" import re -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Set from certbot.plugins import common diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index a3ef2d760..2ab160c2f 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -4,7 +4,7 @@ import logging import pkg_resources import zope.interface -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot import util diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index aae3dc6e4..992672913 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -7,9 +7,9 @@ import sys import six -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict +from acme.magic_typing import List +from acme.magic_typing import Set from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util @@ -321,7 +321,7 @@ class ApacheParser(object): for mod in matches: self.add_mod(mod.strip()) - def filter_args_num(self, matches, args): # pylint: disable=no-self-use + def filter_args_num(self, matches, args): """Filter out directives with specific number of arguments. This function makes the assumption that all related arguments are given @@ -715,7 +715,7 @@ class ApacheParser(object): return get_aug_path(arg) - def fnmatch_to_re(self, clean_fn_match): # pylint: disable=no-self-use + def fnmatch_to_re(self, clean_fn_match): """Method converts Apache's basic fnmatch to regular expression. Assumption - Configs are assumed to be well-formed and only writable by diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py index 3011b9823..7cb4e9722 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py @@ -5,7 +5,7 @@ import subprocess import zope.interface -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Set from certbot._internal import configuration from certbot_compatibility_test import errors from certbot_compatibility_test import interfaces diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index 2c3f880e0..d719f583b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -15,8 +15,8 @@ from urllib3.util import connection from acme import challenges from acme import crypto_util from acme import messages -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List +from acme.magic_typing import Tuple from certbot import achallenges from certbot import errors as le_errors from certbot.tests import acme_util diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index 796ebbe9d..b527ce16b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -4,7 +4,7 @@ import socket import requests import six -from six.moves import xrange # pylint: disable=import-error, redefined-builtin +from six.moves import xrange from acme import crypto_util from acme import errors as acme_errors @@ -13,7 +13,6 @@ logger = logging.getLogger(__name__) class Validator(object): - # pylint: disable=no-self-use """Collection of functions to test a live webserver's configuration""" def certificate(self, cert, name, alt_host=None, port=443): diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index 22124ac04..22dbcfa1f 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -38,7 +38,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add) add('credentials', help='Cloudflare credentials INI file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the Cloudflare API.' diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py index 2a0f12ea7..654c04c70 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py @@ -34,7 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='CloudXNS credentials INI file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the CloudXNS API.' diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index 7f3abbe31..75e25a848 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -30,7 +30,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add) add('credentials', help='DigitalOcean credentials INI file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the DigitalOcean API.' diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py index 8c48d31e7..9f7f100d7 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py @@ -34,7 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='DNSimple credentials INI file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the DNSimple API.' diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py index ed3146dce..4a1fcffc3 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py @@ -35,7 +35,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help='DNS Made Easy credentials INI file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the DNS Made Easy API.' diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py index 76c0ed584..39deddae5 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py @@ -34,7 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='Gehirn Infrastructure Service credentials file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the Gehirn Infrastructure Service API.' diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index 3aa910b52..4eaed1783 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -45,7 +45,7 @@ class Authenticator(dns_common.DNSAuthenticator): 'required permissions.)').format(ACCT_URL, PERMISSIONS_URL), default=None) - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the Google Cloud DNS API.' @@ -148,7 +148,7 @@ class _GoogleClient(object): }, ] - changes = self.dns.changes() # changes | pylint: disable=no-member + changes = self.dns.changes() try: request = changes.create(project=self.project_id, managedZone=zone_id, body=data) @@ -213,7 +213,7 @@ class _GoogleClient(object): }, ] - changes = self.dns.changes() # changes | pylint: disable=no-member + changes = self.dns.changes() try: request = changes.create(project=self.project_id, managedZone=zone_id, body=data) @@ -264,7 +264,7 @@ class _GoogleClient(object): zone_dns_name_guesses = dns_common.base_domain_name_guesses(domain) - mz = self.dns.managedZones() # managedZones | pylint: disable=no-member + mz = self.dns.managedZones() for zone_name in zone_dns_name_guesses: try: request = mz.list(project=self.project_id, dnsName=zone_name + '.') diff --git a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py index ea6046849..f7b3ec3d4 100644 --- a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py @@ -35,7 +35,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=1200) add('credentials', help='Linode credentials INI file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the Linode API.' diff --git a/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py b/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py index 7c18c7131..d5b499c72 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py +++ b/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py @@ -34,7 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='LuaDNS credentials INI file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the LuaDNS API.' diff --git a/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py b/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py index f5af37389..d328d80ce 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py +++ b/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py @@ -34,7 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='NS1 credentials file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the NS1 API.' diff --git a/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py index a495983f2..11ae6b8f0 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py +++ b/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py @@ -34,7 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='OVH credentials INI file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the OVH API.' diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py index cb4d5addb..3bb4f444b 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py @@ -50,7 +50,7 @@ class Authenticator(dns_common.DNSAuthenticator): super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help='RFC 2136 credentials INI file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'RFC 2136 Dynamic Updates.' diff --git a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py index 637558304..6250d2274 100644 --- a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py @@ -8,9 +8,9 @@ from botocore.exceptions import ClientError from botocore.exceptions import NoCredentialsError import zope.interface -from acme.magic_typing import DefaultDict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import DefaultDict +from acme.magic_typing import Dict +from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot.plugins import dns_common @@ -41,13 +41,13 @@ class Authenticator(dns_common.DNSAuthenticator): self.r53 = boto3.client("route53") self._resource_records = collections.defaultdict(list) # type: DefaultDict[str, List[Dict[str, str]]] - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return "Solve a DNS01 challenge using AWS Route53" def _setup_credentials(self): pass - def _perform(self, domain, validation_name, validation): # pylint: disable=missing-docstring + def _perform(self, domain, validation_name, validation): pass def perform(self, achalls): diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py index 25042bfc6..67cfb2e97 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py @@ -35,7 +35,7 @@ class Authenticator(dns_common.DNSAuthenticator): add, default_propagation_seconds=90) add('credentials', help='Sakura Cloud credentials file.') - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ 'the Sakura Cloud API.' diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 52d8a08bc..459950aa1 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -1,6 +1,6 @@ """Nginx Configuration""" # https://github.com/PyCQA/pylint/issues/73 -from distutils.version import LooseVersion # pylint: disable=no-name-in-module, import-error +from distutils.version import LooseVersion import logging import re import socket @@ -14,9 +14,9 @@ import zope.interface from acme import challenges from acme import crypto_util as acme_crypto_util -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict +from acme.magic_typing import List +from acme.magic_typing import Set from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -696,7 +696,7 @@ class NginxConfigurator(common.Installer): ################################## # enhancement methods (IInstaller) ################################## - def supported_enhancements(self): # pylint: disable=no-self-use + def supported_enhancements(self): """Returns currently supported enhancements.""" return ['redirect', 'ensure-http-header', 'staple-ocsp'] @@ -915,7 +915,7 @@ class NginxConfigurator(common.Installer): """ nginx_restart(self.conf('ctl'), self.nginx_conf) - def config_test(self): # pylint: disable=no-self-use + def config_test(self): """Check the configuration of Nginx for errors. :raises .errors.MisconfigurationError: If config_test fails @@ -1090,7 +1090,7 @@ class NginxConfigurator(common.Installer): ########################################################################### # Challenges Section for IAuthenticator ########################################################################### - def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use + def get_chall_pref(self, unused_domain): """Return list of challenge preferences.""" return [challenges.HTTP01] diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index 97b111576..2f6458f87 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -3,7 +3,7 @@ import logging from acme import challenges -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List from certbot import errors from certbot.compat import os from certbot.plugins import common diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index 4fa1362a0..a8ac90427 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -20,7 +20,6 @@ import six logger = logging.getLogger(__name__) class RawNginxParser(object): - # pylint: disable=expression-not-assigned # pylint: disable=pointless-statement """A class that parses nginx configuration with pyparsing.""" diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index edb77a1c1..f71d7c018 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -8,11 +8,11 @@ import re import pyparsing import six -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict +from acme.magic_typing import List +from acme.magic_typing import Set +from acme.magic_typing import Tuple +from acme.magic_typing import Union from certbot import errors from certbot.compat import os from certbot_nginx._internal import nginxparser @@ -127,7 +127,6 @@ class NginxParser(object): return servers def get_vhosts(self): - # pylint: disable=cell-var-from-loop """Gets list of all 'virtual hosts' found in Nginx configuration. Technically this is a misnomer because Nginx does not have virtual hosts, it has 'server blocks'. diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index e03913887..61b31b2d5 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -6,7 +6,7 @@ import logging import six -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List from certbot import errors logger = logging.getLogger(__name__) diff --git a/certbot/certbot/_internal/auth_handler.py b/certbot/certbot/_internal/auth_handler.py index e4ad91247..7ea2a1de8 100644 --- a/certbot/certbot/_internal/auth_handler.py +++ b/certbot/certbot/_internal/auth_handler.py @@ -8,9 +8,9 @@ import zope.component from acme import challenges from acme import errors as acme_errors from acme import messages -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict +from acme.magic_typing import List +from acme.magic_typing import Tuple from certbot import achallenges from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index 298e7d269..e6cbd5c2c 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -7,7 +7,7 @@ import traceback import pytz import zope.component -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List from certbot import crypto_util from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/cli.py b/certbot/certbot/_internal/cli.py index d5d498b4d..c0dbf7424 100644 --- a/certbot/certbot/_internal/cli.py +++ b/certbot/certbot/_internal/cli.py @@ -15,9 +15,9 @@ import zope.interface from zope.interface import interfaces as zope_interfaces from acme import challenges -from acme.magic_typing import Any # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any +from acme.magic_typing import Dict +from acme.magic_typing import Optional import certbot from certbot import crypto_util from certbot import errors @@ -505,7 +505,7 @@ class HelpfulArgumentParser(object): " and ".join(flag_default("config_files")))) # This is the only way to turn off overly verbose config flag documentation - self.parser._add_config_file_help = False # pylint: disable=protected-access + self.parser._add_config_file_help = False # Help that are synonyms for --help subcommands COMMANDS_TOPICS = ["command", "commands", "subcommand", "subcommands", "verbs"] diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index 9ce741e38..526f4200f 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -14,8 +14,8 @@ from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors from acme import messages -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List +from acme.magic_typing import Optional import certbot from certbot import crypto_util from certbot import errors @@ -343,7 +343,7 @@ class Client(object): orderr = self._get_order_and_authorizations(csr.data, self.config.allow_subset_of_names) authzr = orderr.authorizations - auth_domains = set(a.body.identifier.value for a in authzr) # pylint: disable=not-an-iterable + auth_domains = set(a.body.identifier.value for a in authzr) successful_domains = [d for d in domains if d in auth_domains] # allow_subset_of_names is currently disabled for wildcard diff --git a/certbot/certbot/_internal/configuration.py b/certbot/certbot/_internal/configuration.py index f3db207db..f1e85f9fe 100644 --- a/certbot/certbot/_internal/configuration.py +++ b/certbot/certbot/_internal/configuration.py @@ -65,7 +65,7 @@ class NamespaceConfig(object): return (parsed.netloc + parsed.path).replace('/', os.path.sep) @property - def accounts_dir(self): # pylint: disable=missing-docstring + def accounts_dir(self): # pylint: disable=missing-function-docstring return self.accounts_dir_for_server_path(self.server_path) def accounts_dir_for_server_path(self, server_path): @@ -75,23 +75,23 @@ class NamespaceConfig(object): self.namespace.config_dir, constants.ACCOUNTS_DIR, server_path) @property - def backup_dir(self): # pylint: disable=missing-docstring + def backup_dir(self): # pylint: disable=missing-function-docstring return os.path.join(self.namespace.work_dir, constants.BACKUP_DIR) @property - def csr_dir(self): # pylint: disable=missing-docstring + def csr_dir(self): # pylint: disable=missing-function-docstring return os.path.join(self.namespace.config_dir, constants.CSR_DIR) @property - def in_progress_dir(self): # pylint: disable=missing-docstring + def in_progress_dir(self): # pylint: disable=missing-function-docstring return os.path.join(self.namespace.work_dir, constants.IN_PROGRESS_DIR) @property - def key_dir(self): # pylint: disable=missing-docstring + def key_dir(self): # pylint: disable=missing-function-docstring return os.path.join(self.namespace.config_dir, constants.KEY_DIR) @property - def temp_checkpoint_dir(self): # pylint: disable=missing-docstring + def temp_checkpoint_dir(self): # pylint: disable=missing-function-docstring return os.path.join( self.namespace.work_dir, constants.TEMP_CHECKPOINT_DIR) @@ -102,15 +102,15 @@ class NamespaceConfig(object): return type(self)(new_ns) @property - def default_archive_dir(self): # pylint: disable=missing-docstring + def default_archive_dir(self): # pylint: disable=missing-function-docstring return os.path.join(self.namespace.config_dir, constants.ARCHIVE_DIR) @property - def live_dir(self): # pylint: disable=missing-docstring + def live_dir(self): # pylint: disable=missing-function-docstring return os.path.join(self.namespace.config_dir, constants.LIVE_DIR) @property - def renewal_configs_dir(self): # pylint: disable=missing-docstring + def renewal_configs_dir(self): # pylint: disable=missing-function-docstring return os.path.join( self.namespace.config_dir, constants.RENEWAL_CONFIGS_DIR) diff --git a/certbot/certbot/_internal/error_handler.py b/certbot/certbot/_internal/error_handler.py index 5ca3cc57e..41c12eafa 100644 --- a/certbot/certbot/_internal/error_handler.py +++ b/certbot/certbot/_internal/error_handler.py @@ -4,11 +4,11 @@ import logging import signal import traceback -from acme.magic_typing import Any # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Callable # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any +from acme.magic_typing import Callable +from acme.magic_typing import Dict +from acme.magic_typing import List +from acme.magic_typing import Union from certbot import errors from certbot.compat import os diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index 25addd915..589c59e89 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -5,8 +5,8 @@ import logging from subprocess import PIPE from subprocess import Popen -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List +from acme.magic_typing import Set from certbot import errors from certbot import util from certbot.compat import filesystem diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index 7823eaac3..8f3c28006 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -2,15 +2,15 @@ import errno import logging -from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Optional from certbot import errors from certbot.compat import filesystem from certbot.compat import os try: - import fcntl # pylint: disable=import-error + import fcntl except ImportError: - import msvcrt # pylint: disable=import-error + import msvcrt POSIX_MODE = False else: POSIX_MODE = True @@ -115,10 +115,10 @@ class _BaseLockMechanism(object): """ return self._fd is not None - def acquire(self): # pylint: disable=missing-docstring + def acquire(self): # pylint: disable=missing-function-docstring pass # pragma: no cover - def release(self): # pylint: disable=missing-docstring + def release(self): # pylint: disable=missing-function-docstring pass # pragma: no cover diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 72fcfca71..8674cd151 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -11,7 +11,7 @@ import josepy as jose import zope.component from acme import errors as acme_errors -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Union import certbot from certbot import crypto_util from certbot import errors diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index d7d6390f7..d98a4cb0c 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -8,7 +8,7 @@ import six import zope.interface import zope.interface.verify -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict from certbot import errors from certbot import interfaces from certbot._internal import constants @@ -195,14 +195,12 @@ class PluginsRegistry(Mapping): # Pylint checks for super init, but also claims the super # has no __init__member - # pylint: disable=super-init-not-called self._plugins = collections.OrderedDict(sorted(six.iteritems(plugins))) @classmethod def find_all(cls): """Find plugins using setuptools entry points.""" plugins = {} # type: Dict[str, PluginEntryPoint] - # pylint: disable=not-callable entry_points = itertools.chain( pkg_resources.iter_entry_points( constants.SETUPTOOLS_PLUGINS_ENTRY_POINT), @@ -212,7 +210,6 @@ class PluginsRegistry(Mapping): plugin_ep = PluginEntryPoint(entry_point) assert plugin_ep.name not in plugins, ( "PREFIX_FREE_DISTRIBUTIONS messed up") - # providedBy | pylint: disable=no-member if interfaces.IPluginFactory.providedBy(plugin_ep.plugin_cls): plugins[plugin_ep.name] = plugin_ep else: # pragma: no cover diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index be6abaad4..3204fe1da 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -3,7 +3,7 @@ import zope.component import zope.interface from acme import challenges -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Dict from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces @@ -81,7 +81,7 @@ permitted by DNS standards.) add('public-ip-logging-ok', action='store_true', help='Automatically allows public IP logging (default: Ask)') - def prepare(self): # pylint: disable=missing-docstring + def prepare(self): # pylint: disable=missing-function-docstring if self.config.noninteractive_mode and not self.conf('auth-hook'): raise errors.PluginError( 'An authentication script must be provided with --{0} when ' @@ -97,17 +97,17 @@ permitted by DNS standards.) hook_prefix = self.option_name(name)[:-len('-hook')] hooks.validate_hook(hook, hook_prefix) - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return ( 'This plugin allows the user to customize setup for domain ' 'validation challenges either through shell scripts provided by ' 'the user or by performing the setup manually.') def get_chall_pref(self, domain): - # pylint: disable=missing-docstring,no-self-use,unused-argument + # pylint: disable=unused-argument,missing-function-docstring return [challenges.HTTP01, challenges.DNS01] - def perform(self, achalls): # pylint: disable=missing-docstring + def perform(self, achalls): # pylint: disable=missing-function-docstring self._verify_ip_logging_ok() if self.conf('auth-hook'): perform_achall = self._perform_achall_with_script @@ -170,7 +170,7 @@ permitted by DNS standards.) display.notification(msg, wrap=False, force_interactive=True) self.subsequent_any_challenge = True - def cleanup(self, achalls): # pylint: disable=missing-docstring + def cleanup(self, achalls): # pylint: disable=missing-function-docstring if self.conf('cleanup-hook'): for achall in achalls: env = self.env.pop(achall) diff --git a/certbot/certbot/_internal/plugins/null.py b/certbot/certbot/_internal/plugins/null.py index bf4615497..cf7c05a2b 100644 --- a/certbot/certbot/_internal/plugins/null.py +++ b/certbot/certbot/_internal/plugins/null.py @@ -18,7 +18,7 @@ class Installer(common.Plugin): description = "Null Installer" hidden = True - # pylint: disable=missing-docstring,no-self-use + # pylint: disable=missing-function-docstring def prepare(self): pass # pragma: no cover diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index 80421299e..bbb56178c 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -11,12 +11,12 @@ import zope.interface from acme import challenges from acme import standalone as acme_standalone -from acme.magic_typing import DefaultDict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import TYPE_CHECKING # pylint: disable=unused-import, no-name-in-module -from certbot import achallenges # pylint: disable=unused-import +from acme.magic_typing import DefaultDict +from acme.magic_typing import Dict +from acme.magic_typing import Set +from acme.magic_typing import Tuple +from acme.magic_typing import TYPE_CHECKING +from certbot import achallenges from certbot import errors from certbot import interfaces from certbot.plugins import common @@ -139,20 +139,20 @@ class Authenticator(common.Plugin): def add_parser_arguments(cls, add): pass # No additional argument for the standalone plugin parser - def more_info(self): # pylint: disable=missing-docstring + def more_info(self): # pylint: disable=missing-function-docstring return("This authenticator creates its own ephemeral TCP listener " "on the necessary port in order to respond to incoming " "http-01 challenges from the certificate authority. Therefore, " "it does not rely on any existing server program.") - def prepare(self): # pylint: disable=missing-docstring + def prepare(self): # pylint: disable=missing-function-docstring pass def get_chall_pref(self, domain): - # pylint: disable=unused-argument,missing-docstring + # pylint: disable=unused-argument,missing-function-docstring return [challenges.HTTP01] - def perform(self, achalls): # pylint: disable=missing-docstring + def perform(self, achalls): # pylint: disable=missing-function-docstring return [self._try_perform_single(achall) for achall in achalls] def _try_perform_single(self, achall): @@ -177,7 +177,7 @@ class Authenticator(common.Plugin): self.http_01_resources.add(resource) return servers, response - def cleanup(self, achalls): # pylint: disable=missing-docstring + def cleanup(self, achalls): # pylint: disable=missing-function-docstring # reduce self.served and close servers if no challenges are served for unused_servers, server_achalls in self.served.items(): for achall in achalls: diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index c7737e0d1..042b60656 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -9,11 +9,11 @@ import six import zope.component import zope.interface -from acme import challenges # pylint: disable=unused-import -from acme.magic_typing import DefaultDict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme import challenges +from acme.magic_typing import DefaultDict +from acme.magic_typing import Dict +from acme.magic_typing import List +from acme.magic_typing import Set from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces @@ -42,7 +42,7 @@ necessary validation resources to appropriate paths on the file system. It expects that there is some other HTTP server configured to serve all files under specified web root ({0}).""" - def more_info(self): # pylint: disable=missing-docstring,no-self-use + def more_info(self): # pylint: disable=missing-function-docstring return self.MORE_INFO.format(self.conf("path")) @classmethod @@ -64,7 +64,7 @@ to serve all files under specified web root ({0}).""" '{"example.com":"/var/www"}.') def get_chall_pref(self, domain): # pragma: no cover - # pylint: disable=missing-docstring,no-self-use,unused-argument + # pylint: disable=unused-argument,missing-function-docstring return [challenges.HTTP01] def __init__(self, *args, **kwargs): @@ -75,10 +75,10 @@ to serve all files under specified web root ({0}).""" # stack of dirs successfully created by this authenticator self._created_dirs = [] # type: List[str] - def prepare(self): # pylint: disable=missing-docstring + def prepare(self): # pylint: disable=missing-function-docstring pass - def perform(self, achalls): # pylint: disable=missing-docstring + def perform(self, achalls): # pylint: disable=missing-function-docstring self._set_webroots(achalls) self._create_challenge_dirs() @@ -213,7 +213,7 @@ to serve all files under specified web root ({0}).""" self.performed[root_path].add(achall) return response - def cleanup(self, achalls): # pylint: disable=missing-docstring + def cleanup(self, achalls): # pylint: disable=missing-function-docstring for achall in achalls: root_path = self.full_roots.get(achall.domain, None) if root_path is not None: diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index bf30404f5..fd23b0d18 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -13,7 +13,7 @@ import OpenSSL import six import zope.component -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List from certbot import crypto_util from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index 947f343d4..3b959f4e1 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -6,7 +6,7 @@ import logging import sys import textwrap -from six.moves import queue # type: ignore # pylint: disable=import-error +from six.moves import queue # type: ignore import zope.interface from certbot import interfaces diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 964515eee..6a34355a8 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -883,7 +883,7 @@ class RenewableCert(interfaces.RenewableCert): return crypto_util.get_names_from_cert(f.read()) def ocsp_revoked(self, version=None): - # pylint: disable=no-self-use,unused-argument + # pylint: disable=unused-argument """Is the specified cert version revoked according to OCSP? Also returns True if the cert version is declared as intended diff --git a/certbot/certbot/compat/_path.py b/certbot/certbot/compat/_path.py index 5c5fe460e..44c5e300f 100644 --- a/certbot/certbot/compat/_path.py +++ b/certbot/certbot/compat/_path.py @@ -9,7 +9,7 @@ from __future__ import absolute_import # First round of wrapping: we import statically all public attributes exposed by the os.path # module. This allows in particular to have pylint, mypy, IDEs be aware that most of os.path # members are available in certbot.compat.path. -from os.path import * # type: ignore # pylint: disable=wildcard-import,unused-wildcard-import,redefined-builtin,os-module-forbidden +from os.path import * # type: ignore # pylint: disable=wildcard-import,unused-wildcard-import,os-module-forbidden # Second round of wrapping: we import dynamically all attributes from the os.path module that have # not yet been imported by the first round (static star import). diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index 65bb53f38..88c2916fa 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -5,12 +5,11 @@ import errno import os # pylint: disable=os-module-forbidden import stat -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List +from acme.magic_typing import Tuple # pylint: disable=unused-import +from acme.magic_typing import Union # pylint: disable=unused-import try: - # pylint: disable=import-error import ntsecuritycon import win32security import win32con diff --git a/certbot/certbot/compat/misc.py b/certbot/certbot/compat/misc.py index ffe611edb..956c56370 100644 --- a/certbot/certbot/compat/misc.py +++ b/certbot/certbot/compat/misc.py @@ -11,7 +11,7 @@ from certbot import errors from certbot.compat import os try: - from win32com.shell import shell as shellwin32 # pylint: disable=import-error + from win32com.shell import shell as shellwin32 POSIX_MODE = False except ImportError: # pragma: no cover POSIX_MODE = True diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 9aae75991..9136445bc 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -23,7 +23,7 @@ import six import zope.component from acme import crypto_util as acme_crypto_util -from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import IO # pylint: disable=unused-import from certbot import errors from certbot import interfaces from certbot import util diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index ba2dd4ecf..05330b1a9 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -334,7 +334,6 @@ class FileDisplay(object): return self.input(message, default, cli_flag, force_interactive) def _scrub_checklist_input(self, indices, tags): - # pylint: disable=no-self-use """Validate input and transform indices to appropriate tags. :param list indices: input diff --git a/certbot/certbot/interfaces.py b/certbot/certbot/interfaces.py index e96712d23..81b41d1ec 100644 --- a/certbot/certbot/interfaces.py +++ b/certbot/certbot/interfaces.py @@ -4,7 +4,7 @@ import abc import six import zope.interface -# pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class +# pylint: disable=no-self-argument,no-method-argument,inherit-non-class @six.add_metaclass(abc.ABCMeta) diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 5c3cfdd59..6a95f26fa 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -16,8 +16,8 @@ from cryptography.hazmat.primitives import serialization import pytz import requests -from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Optional +from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import util @@ -26,7 +26,7 @@ from certbot.interfaces import RenewableCert # pylint: disable=unused-import try: # Only cryptography>=2.5 has ocsp module # and signature_hash_algorithm attribute in OCSPResponse class - from cryptography.x509 import ocsp # pylint: disable=import-error, ungrouped-imports + from cryptography.x509 import ocsp # pylint: disable=ungrouped-imports getattr(ocsp.OCSPResponse, 'signature_hash_algorithm') except (ImportError, AttributeError): # pragma: no cover ocsp = None # type: ignore diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index 6fa1e76f8..7d21e6d0a 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -10,7 +10,7 @@ from josepy import util as jose_util import pkg_resources import zope.interface -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import List from certbot import achallenges # pylint: disable=unused-import from certbot import crypto_util from certbot import errors @@ -74,7 +74,6 @@ class Plugin(object): """ # dummy function, doesn't check if dest.startswith(self.dest_namespace) def add(arg_name_no_prefix, *args, **kwargs): - # pylint: disable=missing-docstring return parser.add_argument( "--{0}{1}".format(option_namespace(name), arg_name_no_prefix), *args, **kwargs) diff --git a/certbot/certbot/plugins/dns_common.py b/certbot/certbot/plugins/dns_common.py index d31266434..245b7dc05 100644 --- a/certbot/certbot/plugins/dns_common.py +++ b/certbot/certbot/plugins/dns_common.py @@ -37,13 +37,13 @@ class DNSAuthenticator(common.Plugin): help='The number of seconds to wait for DNS to propagate before asking the ACME server ' 'to verify the DNS record.') - def get_chall_pref(self, unused_domain): # pylint: disable=missing-docstring,no-self-use + def get_chall_pref(self, unused_domain): # pylint: disable=missing-function-docstring return [challenges.DNS01] - def prepare(self): # pylint: disable=missing-docstring + def prepare(self): # pylint: disable=missing-function-docstring pass - def perform(self, achalls): # pylint: disable=missing-docstring + def perform(self, achalls): # pylint: disable=missing-function-docstring self._setup_credentials() self._attempt_cleanup = True @@ -66,7 +66,7 @@ class DNSAuthenticator(common.Plugin): return responses - def cleanup(self, achalls): # pylint: disable=missing-docstring + def cleanup(self, achalls): # pylint: disable=missing-function-docstring if self._attempt_cleanup: for achall in achalls: domain = achall.domain diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index 3e28a291b..c3d80ca29 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -4,9 +4,9 @@ import logging from requests.exceptions import HTTPError from requests.exceptions import RequestException -from acme.magic_typing import Any # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any +from acme.magic_typing import Dict +from acme.magic_typing import Union from certbot import errors from certbot.plugins import dns_common diff --git a/certbot/certbot/plugins/enhancements.py b/certbot/certbot/plugins/enhancements.py index f8d9db7dc..be9b7933d 100644 --- a/certbot/certbot/plugins/enhancements.py +++ b/certbot/certbot/plugins/enhancements.py @@ -3,9 +3,9 @@ import abc import six -from acme.magic_typing import Any # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any +from acme.magic_typing import Dict +from acme.magic_typing import List from certbot._internal import constants ENHANCEMENTS = ["redirect", "ensure-http-header", "ocsp-stapling"] diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index 7956295d2..fe3a97dd5 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -2,8 +2,8 @@ import json import logging -from acme.magic_typing import Any # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any +from acme.magic_typing import Dict from certbot import errors from certbot.compat import filesystem from certbot.compat import os diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index 47a77c80a..80ba0f599 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -195,7 +195,7 @@ class Reverter(object): with open(os.path.join(cp_dir, "CHANGES_SINCE"), "a") as notes_fd: notes_fd.write(save_notes) - def _read_and_append(self, filepath): # pylint: disable=no-self-use + def _read_and_append(self, filepath): """Reads the file lines and returns a file obj. Read the file returning the lines, and a pointer to the end of the file. @@ -250,7 +250,7 @@ class Reverter(object): raise errors.ReverterError( "Unable to remove directory: %s" % cp_dir) - def _run_undo_commands(self, filepath): # pylint: disable=no-self-use + def _run_undo_commands(self, filepath): """Run all commands in a file.""" # NOTE: csv module uses native strings. That is, bytes on Python 2 and # unicode on Python 3 @@ -413,7 +413,7 @@ class Reverter(object): "Incomplete or failed recovery for IN_PROGRESS checkpoint " "- %s" % self.config.in_progress_dir) - def _remove_contained_files(self, file_list): # pylint: disable=no-self-use + def _remove_contained_files(self, file_list): """Erase all files contained within file_list. :param str file_list: file containing list of file paths to be deleted diff --git a/certbot/certbot/tests/acme_util.py b/certbot/certbot/tests/acme_util.py index 3d560dcbc..f4a20ea86 100644 --- a/certbot/certbot/tests/acme_util.py +++ b/certbot/certbot/tests/acme_util.py @@ -27,7 +27,7 @@ def gen_combos(challbs): return tuple((i,) for i, _ in enumerate(challbs)) -def chall_to_challb(chall, status): # pylint: disable=redefined-outer-name +def chall_to_challb(chall, status): """Return ChallengeBody from Challenge.""" kwargs = { "chall": chall, @@ -67,7 +67,6 @@ def gen_authzr(authz_status, domain, challs, statuses, combos=True): :param bool combos: Whether or not to add combinations """ - # pylint: disable=redefined-outer-name challbs = tuple( chall_to_challb(chall, status) for chall, status in six.moves.zip(challs, statuses) diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index a9870b0fd..8b28b1080 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -14,7 +14,7 @@ import mock import OpenSSL import pkg_resources import six -from six.moves import reload_module # pylint: disable=import-error +from six.moves import reload_module from certbot import interfaces from certbot import util diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index 0a47cd87a..aff2952f7 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -5,7 +5,7 @@ import argparse import atexit import collections from collections import OrderedDict -import distutils.version # pylint: disable=import-error,no-name-in-module +import distutils.version import errno import logging import platform @@ -17,8 +17,8 @@ import sys import configargparse import six -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Tuple +from acme.magic_typing import Union from certbot import errors from certbot._internal import constants from certbot._internal import lock @@ -26,7 +26,7 @@ from certbot.compat import filesystem from certbot.compat import os if sys.platform.startswith('linux'): - import distro # pylint: disable=import-error + import distro _USE_DISTRO = True else: _USE_DISTRO = False diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index ebe4e3671..a5947399a 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -16,7 +16,7 @@ import textwrap import six -from letshelp_certbot.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from letshelp_certbot.magic_typing import List _DESCRIPTION = """ Let's Help is a simple script you can run to help out the Certbot diff --git a/letshelp-certbot/letshelp_certbot/magic_typing.py b/letshelp-certbot/letshelp_certbot/magic_typing.py index 5a6358c69..d6b1ff056 100644 --- a/letshelp-certbot/letshelp_certbot/magic_typing.py +++ b/letshelp-certbot/letshelp_certbot/magic_typing.py @@ -10,7 +10,6 @@ class TypingClass(object): try: # mypy doesn't respect modifying sys.modules from typing import * # pylint: disable=wildcard-import, unused-wildcard-import - # pylint: disable=unused-import from typing import Collection, IO # type: ignore # pylint: enable=unused-import except ImportError: diff --git a/letshelp-certbot/letshelp_certbot/magic_typing_test.py b/letshelp-certbot/letshelp_certbot/magic_typing_test.py index 200ca03b8..41bc245fa 100644 --- a/letshelp-certbot/letshelp_certbot/magic_typing_test.py +++ b/letshelp-certbot/letshelp_certbot/magic_typing_test.py @@ -18,7 +18,7 @@ class MagicTypingTest(unittest.TestCase): sys.modules['typing'] = typing_class_mock if 'letshelp_certbot.magic_typing' in sys.modules: del sys.modules['letshelp_certbot.magic_typing'] # pragma: no cover - from letshelp_certbot.magic_typing import Text # pylint: disable=no-name-in-module + from letshelp_certbot.magic_typing import Text self.assertEqual(Text, text_mock) del sys.modules['letshelp_certbot.magic_typing'] sys.modules['typing'] = temp_typing @@ -31,7 +31,7 @@ class MagicTypingTest(unittest.TestCase): sys.modules['typing'] = None if 'letshelp_certbot.magic_typing' in sys.modules: del sys.modules['letshelp_certbot.magic_typing'] # pragma: no cover - from letshelp_certbot.magic_typing import Text # pylint: disable=no-name-in-module + from letshelp_certbot.magic_typing import Text self.assertTrue(Text is None) del sys.modules['letshelp_certbot.magic_typing'] sys.modules['typing'] = temp_typing From 3f52695ec27a4b5fdfe675982e959f7fffbccb18 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 14 Feb 2020 17:18:53 -0800 Subject: [PATCH 10/92] more robustly stop patches (#7763) --- certbot/tests/cert_manager_test.py | 8 +++--- certbot/tests/main_test.py | 40 +++++++++++++----------------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 81134f02f..eb8005b2b 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -564,14 +564,12 @@ class GetCertnameTest(unittest.TestCase): """Tests for certbot._internal.cert_manager.""" def setUp(self): - self.get_utility_patch = test_util.patch_get_utility() - self.mock_get_utility = self.get_utility_patch.start() + get_utility_patch = test_util.patch_get_utility() + self.mock_get_utility = get_utility_patch.start() + self.addCleanup(get_utility_patch.stop) self.config = mock.MagicMock() self.config.certname = None - def tearDown(self): - self.get_utility_patch.stop() - @mock.patch('certbot._internal.storage.renewal_conf_files') @mock.patch('certbot._internal.storage.lineagename_for_filename') def test_get_certnames(self, mock_name, mock_files): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 7b22c81d6..7ebe5e66a 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -62,7 +62,7 @@ class RunTest(test_util.ConfigTestCase): def setUp(self): super(RunTest, self).setUp() self.domain = 'example.org' - self.patches = [ + patches = [ mock.patch('certbot._internal.main._get_and_save_cert'), mock.patch('certbot._internal.main.display_ops.success_installation'), mock.patch('certbot._internal.main.display_ops.success_renewal'), @@ -71,17 +71,15 @@ class RunTest(test_util.ConfigTestCase): mock.patch('certbot._internal.main._report_new_cert'), mock.patch('certbot._internal.main._find_cert')] - self.mock_auth = self.patches[0].start() - self.mock_success_installation = self.patches[1].start() - self.mock_success_renewal = self.patches[2].start() - self.mock_init = self.patches[3].start() - self.mock_suggest_donation = self.patches[4].start() - self.mock_report_cert = self.patches[5].start() - self.mock_find_cert = self.patches[6].start() - - def tearDown(self): - for patch in self.patches: - patch.stop() + self.mock_auth = patches[0].start() + self.mock_success_installation = patches[1].start() + self.mock_success_renewal = patches[2].start() + self.mock_init = patches[3].start() + self.mock_suggest_donation = patches[4].start() + self.mock_report_cert = patches[5].start() + self.mock_find_cert = patches[6].start() + for patch in patches: + self.addCleanup(patch.stop) def _call(self): args = '-a webroot -i null -d {0}'.format(self.domain).split() @@ -243,16 +241,18 @@ class RevokeTest(test_util.TempDirTestCase): with open(self.tmp_cert_path, 'r') as f: self.tmp_cert = (self.tmp_cert_path, f.read()) - self.patches = [ + patches = [ mock.patch('acme.client.BackwardsCompatibleClientV2'), mock.patch('certbot._internal.client.Client'), mock.patch('certbot._internal.main._determine_account'), mock.patch('certbot._internal.main.display_ops.success_revocation') ] - self.mock_acme_client = self.patches[0].start() - self.patches[1].start() - self.mock_determine_account = self.patches[2].start() - self.mock_success_revoke = self.patches[3].start() + self.mock_acme_client = patches[0].start() + patches[1].start() + self.mock_determine_account = patches[2].start() + self.mock_success_revoke = patches[3].start() + for patch in patches: + self.addCleanup(patch.stop) from certbot._internal.account import Account @@ -265,12 +265,6 @@ class RevokeTest(test_util.TempDirTestCase): self.mock_determine_account.return_value = (self.acc, None) - def tearDown(self): - super(RevokeTest, self).tearDown() - - for patch in self.patches: - patch.stop() - def _call(self, args=None): if not args: args = 'revoke --cert-path={0} ' From fd64c8c33b2176e6569d64d30776bd5fc9fd3820 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 14 Feb 2020 17:19:19 -0800 Subject: [PATCH 11/92] Remove letshelp-certbot (#7761) * remove references to letshelp * remove letshelp files * Remove line continuation Co-authored-by: ohemorange --- certbot/docs/packaging.rst | 2 +- letshelp-certbot/LICENSE.txt | 190 ----------- letshelp-certbot/MANIFEST.in | 4 - letshelp-certbot/README.rst | 1 - letshelp-certbot/docs/.gitignore | 1 - letshelp-certbot/docs/Makefile | 192 ----------- letshelp-certbot/docs/_static/.gitignore | 0 letshelp-certbot/docs/_templates/.gitignore | 0 letshelp-certbot/docs/api.rst | 8 - letshelp-certbot/docs/api/index.rst | 11 - letshelp-certbot/docs/conf.py | 310 ----------------- letshelp-certbot/docs/index.rst | 27 -- letshelp-certbot/docs/make.bat | 263 --------------- letshelp-certbot/letshelp_certbot/__init__.py | 1 - letshelp-certbot/letshelp_certbot/apache.py | 314 ------------------ .../letshelp_certbot/apache_test.py | 241 -------------- .../letshelp_certbot/magic_typing.py | 16 - .../letshelp_certbot/magic_typing_test.py | 41 --- .../testdata/mods-available/ssl.load | 2 - .../testdata/mods-enabled/ssl.load | 1 - .../testdata/super_secret_file.txt | 1 - .../testdata/uncommonly_named_k3y | 6 - .../testdata/uncommonly_named_p4sswd | 1 - .../readthedocs.org.requirements.txt | 10 - letshelp-certbot/setup.cfg | 2 - letshelp-certbot/setup.py | 58 ---- linter_plugin.py | 2 +- mypy.ini | 3 - tools/_venv_common.py | 1 - tools/install_and_test.py | 2 +- tox.cover.py | 5 +- tox.ini | 4 +- 32 files changed, 6 insertions(+), 1714 deletions(-) delete mode 100644 letshelp-certbot/LICENSE.txt delete mode 100644 letshelp-certbot/MANIFEST.in delete mode 100644 letshelp-certbot/README.rst delete mode 100644 letshelp-certbot/docs/.gitignore delete mode 100644 letshelp-certbot/docs/Makefile delete mode 100644 letshelp-certbot/docs/_static/.gitignore delete mode 100644 letshelp-certbot/docs/_templates/.gitignore delete mode 100644 letshelp-certbot/docs/api.rst delete mode 100644 letshelp-certbot/docs/api/index.rst delete mode 100644 letshelp-certbot/docs/conf.py delete mode 100644 letshelp-certbot/docs/index.rst delete mode 100644 letshelp-certbot/docs/make.bat delete mode 100644 letshelp-certbot/letshelp_certbot/__init__.py delete mode 100755 letshelp-certbot/letshelp_certbot/apache.py delete mode 100644 letshelp-certbot/letshelp_certbot/apache_test.py delete mode 100644 letshelp-certbot/letshelp_certbot/magic_typing.py delete mode 100644 letshelp-certbot/letshelp_certbot/magic_typing_test.py delete mode 100644 letshelp-certbot/letshelp_certbot/testdata/mods-available/ssl.load delete mode 120000 letshelp-certbot/letshelp_certbot/testdata/mods-enabled/ssl.load delete mode 100644 letshelp-certbot/letshelp_certbot/testdata/super_secret_file.txt delete mode 100644 letshelp-certbot/letshelp_certbot/testdata/uncommonly_named_k3y delete mode 100644 letshelp-certbot/letshelp_certbot/testdata/uncommonly_named_p4sswd delete mode 100644 letshelp-certbot/readthedocs.org.requirements.txt delete mode 100644 letshelp-certbot/setup.cfg delete mode 100644 letshelp-certbot/setup.py diff --git a/certbot/docs/packaging.rst b/certbot/docs/packaging.rst index 7b0b1d41a..d2a94dbe7 100644 --- a/certbot/docs/packaging.rst +++ b/certbot/docs/packaging.rst @@ -38,7 +38,7 @@ Notes for package maintainers 0. Please use our tagged releases, not ``master``! -1. Do not package ``certbot-compatibility-test`` or ``letshelp-certbot`` - it's only used internally. +1. Do not package ``certbot-compatibility-test`` as it's only used internally. 2. To run tests on our packages, you should use ``python setup.py test``. Doing things like running ``pytest`` directly on our package files may not work because Certbot relies on setuptools to register and find its plugins. diff --git a/letshelp-certbot/LICENSE.txt b/letshelp-certbot/LICENSE.txt deleted file mode 100644 index 981c46c9f..000000000 --- a/letshelp-certbot/LICENSE.txt +++ /dev/null @@ -1,190 +0,0 @@ - Copyright 2015 Electronic Frontier Foundation and others - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS diff --git a/letshelp-certbot/MANIFEST.in b/letshelp-certbot/MANIFEST.in deleted file mode 100644 index 623392f28..000000000 --- a/letshelp-certbot/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -include LICENSE.txt -include README.rst -recursive-include docs * -recursive-include letshelp_certbot/testdata * diff --git a/letshelp-certbot/README.rst b/letshelp-certbot/README.rst deleted file mode 100644 index bbe2f2570..000000000 --- a/letshelp-certbot/README.rst +++ /dev/null @@ -1 +0,0 @@ -Let's help Certbot client diff --git a/letshelp-certbot/docs/.gitignore b/letshelp-certbot/docs/.gitignore deleted file mode 100644 index ba65b13af..000000000 --- a/letshelp-certbot/docs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/_build/ diff --git a/letshelp-certbot/docs/Makefile b/letshelp-certbot/docs/Makefile deleted file mode 100644 index 4b392ab8d..000000000 --- a/letshelp-certbot/docs/Makefile +++ /dev/null @@ -1,192 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " applehelp to make an Apple Help Book" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - @echo " coverage to run coverage check of the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/letshelp-certbot.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/letshelp-certbot.qhc" - -applehelp: - $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp - @echo - @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." - @echo "N.B. You won't be able to view it unless you put it in" \ - "~/Library/Documentation/Help or install it in your application" \ - "bundle." - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/letshelp-certbot" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/letshelp-certbot" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -coverage: - $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage - @echo "Testing of coverage in the sources finished, look at the " \ - "results in $(BUILDDIR)/coverage/python.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/letshelp-certbot/docs/_static/.gitignore b/letshelp-certbot/docs/_static/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/letshelp-certbot/docs/_templates/.gitignore b/letshelp-certbot/docs/_templates/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/letshelp-certbot/docs/api.rst b/letshelp-certbot/docs/api.rst deleted file mode 100644 index 8668ec5d8..000000000 --- a/letshelp-certbot/docs/api.rst +++ /dev/null @@ -1,8 +0,0 @@ -================= -API Documentation -================= - -.. toctree:: - :glob: - - api/** diff --git a/letshelp-certbot/docs/api/index.rst b/letshelp-certbot/docs/api/index.rst deleted file mode 100644 index 5ced5f501..000000000 --- a/letshelp-certbot/docs/api/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -:mod:`letshelp_certbot` ---------------------------- - -.. automodule:: letshelp_certbot - :members: - -:mod:`letshelp_certbot.apache` -================================== - -.. automodule:: letshelp_certbot.apache - :members: diff --git a/letshelp-certbot/docs/conf.py b/letshelp-certbot/docs/conf.py deleted file mode 100644 index b4289a345..000000000 --- a/letshelp-certbot/docs/conf.py +++ /dev/null @@ -1,310 +0,0 @@ -# -*- coding: utf-8 -*- -# -# letshelp-certbot documentation build configuration file, created by -# sphinx-quickstart on Sun Oct 18 13:40:19 2015. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import os -import shlex -import sys - -here = os.path.abspath(os.path.dirname(__file__)) - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath(os.path.join(here, '..'))) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', -] - -autodoc_member_order = 'bysource' -autodoc_default_flags = ['show-inheritance'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'letshelp-certbot' -copyright = u'2014-2015, Let\'s Encrypt Project' -author = u'Certbot Project' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '0' -# The full version, including alpha/beta/rc tags. -release = '0' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = 'en' - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -default_role = 'py:obj' - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. - -# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs -# on_rtd is whether we are on readthedocs.org -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' -if not on_rtd: # only import and set the theme if we're building docs locally - import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -# otherwise, readthedocs.org uses their theme by default, so no need to specify it - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'letshelp-certbotdoc' - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - #'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - #'preamble': '', - - # Latex figure (float) alignment - #'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'letshelp-certbot.tex', u'letshelp-certbot Documentation', - u'Certbot Project', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'letshelp-certbot', u'letshelp-certbot Documentation', - [author], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'letshelp-certbot', u'letshelp-certbot Documentation', - author, 'letshelp-certbot', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False - - -intersphinx_mapping = { - 'python': ('https://docs.python.org/', None), - 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), - 'certbot': ('https://certbot.eff.org/docs/', None), -} diff --git a/letshelp-certbot/docs/index.rst b/letshelp-certbot/docs/index.rst deleted file mode 100644 index 678d9be2e..000000000 --- a/letshelp-certbot/docs/index.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. letshelp-certbot documentation master file, created by - sphinx-quickstart on Sun Oct 18 13:40:19 2015. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to letshelp-certbot's documentation! -================================================ - -Contents: - -.. toctree:: - :maxdepth: 2 - - -.. toctree:: - :maxdepth: 1 - - api - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/letshelp-certbot/docs/make.bat b/letshelp-certbot/docs/make.bat deleted file mode 100644 index 0229b4f69..000000000 --- a/letshelp-certbot/docs/make.bat +++ /dev/null @@ -1,263 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - echo. coverage to run coverage check of the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -REM Check if sphinx-build is available and fallback to Python version if any -%SPHINXBUILD% 2> nul -if errorlevel 9009 goto sphinx_python -goto sphinx_ok - -:sphinx_python - -set SPHINXBUILD=python -m sphinx.__init__ -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -:sphinx_ok - - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\letshelp-certbot.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\letshelp-certbot.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %~dp0 - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %~dp0 - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "coverage" ( - %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage - if errorlevel 1 exit /b 1 - echo. - echo.Testing of coverage in the sources finished, look at the ^ -results in %BUILDDIR%/coverage/python.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end diff --git a/letshelp-certbot/letshelp_certbot/__init__.py b/letshelp-certbot/letshelp_certbot/__init__.py deleted file mode 100644 index 6882a19d4..000000000 --- a/letshelp-certbot/letshelp_certbot/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tools for submitting server configurations""" diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py deleted file mode 100755 index a5947399a..000000000 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python -"""Certbot Apache configuration submission script""" - -from __future__ import print_function - -import argparse -import atexit -import os -import re -import shutil -import subprocess -import sys -import tarfile -import tempfile -import textwrap - -import six - -from letshelp_certbot.magic_typing import List - -_DESCRIPTION = """ -Let's Help is a simple script you can run to help out the Certbot -project. Since Certbot will support automatically configuring HTTPS on -many servers, we want to test this functionality on as many configurations as -possible. This script will create a sanitized copy of your Apache -configuration, notifying you of the files that have been selected. If (and only -if) you approve this selection, these files will be sent to the Certbot -developers. - -""" - - -_NO_APACHECTL = """ -Unable to find `apachectl` which is required for this script to work. If it is -installed, please run this script again with the --apache-ctl command line -argument and the path to the binary. - -""" - - -# Keywords likely to be found in filenames of sensitive files -_SENSITIVE_FILENAME_REGEX = re.compile(r"^(?!.*proxy_fdpass).*pass.*$|private|" - r"secret|^(?!.*certbot).*cert.*$|crt|" - r"key|rsa|dsa|pw|\.pem|\.der|\.p12|" - r"\.pfx|\.p7b") - - -def make_and_verify_selection(server_root, temp_dir): - """Copies server_root to temp_dir and verifies selection with the user - - :param str server_root: Path to the Apache server root - :param str temp_dir: Path to the temporary directory to copy files to - - """ - copied_files, copied_dirs = copy_config(server_root, temp_dir) - - print(textwrap.fill("A secure copy of the files that have been selected " - "for submission has been created under {0}. All " - "comments have been removed and the files are only " - "accessible by the current user. A list of the files " - "that have been included is shown below. Please make " - "sure that this selection does not contain private " - "keys, passwords, or any other sensitive " - "information.".format(temp_dir))) - print("\nFiles:") - for copied_file in copied_files: - print(copied_file) - print("Directories (including all contained files):") - for copied_dir in copied_dirs: - print(copied_dir) - - sys.stdout.write("\nIs it safe to submit these files? ") - while True: - ans = six.moves.input("(Y)es/(N)o: ").lower() - if ans.startswith("y"): - return - if ans.startswith("n"): - sys.exit("Your files were not submitted") - - -def copy_config(server_root, temp_dir): - """Safely copies server_root to temp_dir and returns copied files - - :param str server_root: Absolute path to the Apache server root - :param str temp_dir: Path to the temporary directory to copy files to - - :returns: List of copied files and a list of leaf directories where - all contained files were copied - :rtype: `tuple` of `list` of `str` - - """ - copied_files = [] # type: List[str] - copied_dirs = [] # type: List[str] - dir_len = len(os.path.dirname(server_root)) - - for config_path, config_dirs, config_files in os.walk(server_root): - temp_path = os.path.join(temp_dir, config_path[dir_len + 1:]) - os.mkdir(temp_path) - - copied_all = True - copied_files_in_current_dir = [] - for config_file in config_files: - config_file_path = os.path.join(config_path, config_file) - temp_file_path = os.path.join(temp_path, config_file) - if os.path.islink(config_file_path): - os.symlink(os.readlink(config_file_path), temp_file_path) - elif safe_config_file(config_file_path): - copy_file_without_comments(config_file_path, temp_file_path) - copied_files_in_current_dir.append(config_file_path) - else: - copied_all = False - - # If copied all files in leaf directory - if copied_all and not config_dirs: - copied_dirs.append(config_path) - else: - copied_files += copied_files_in_current_dir - - return copied_files, copied_dirs - - -def copy_file_without_comments(source, destination): - """Copies source to destination, removing comments - - :param str source: Path to the file to be copied - :param str destination: Path where source should be copied to - - """ - with open(source, "r") as infile: - with open(destination, "w") as outfile: - for line in infile: - if not (line.isspace() or line.lstrip().startswith("#")): - outfile.write(line) - - -def safe_config_file(config_file): - """Returns True if config_file can be safely copied - - :param str config_file: Path to an Apache configuration file - - :returns: True if config_file can be safely copied - :rtype: bool - - """ - config_file_lower = config_file.lower() - if _SENSITIVE_FILENAME_REGEX.search(config_file_lower): - return False - - proc = subprocess.Popen(["file", config_file], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - file_output, _ = proc.communicate() - - if "ASCII" in file_output: - possible_password_file = empty_or_all_comments = True - with open(config_file) as config_fd: - for line in config_fd: - if not (line.isspace() or line.lstrip().startswith("#")): - empty_or_all_comments = False - if line.startswith("-----BEGIN"): - return False - if ":" not in line: - possible_password_file = False - # If file isn't empty or commented out and could be a password file, - # don't include it in selection. It is safe to include the file if - # it consists solely of comments because comments are removed before - # submission. - return empty_or_all_comments or not possible_password_file - - return False - - -def setup_tempdir(args): - """Creates a temporary directory and necessary files for config - - :param argparse.Namespace args: Parsed command line arguments - - :returns: Path to temporary directory - :rtype: str - - """ - tempdir = tempfile.mkdtemp() - - with open(os.path.join(tempdir, "config_file"), "w") as config_fd: - config_fd.write(args.config_file + "\n") - - proc = subprocess.Popen([args.apache_ctl, "-v"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - with open(os.path.join(tempdir, "version"), "w") as version_fd: - version_fd.write(proc.communicate()[0]) - - proc = subprocess.Popen([args.apache_ctl, "-d", args.server_root, "-f", - args.config_file, "-M"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - with open(os.path.join(tempdir, "modules"), "w") as modules_fd: - modules_fd.write(proc.communicate()[0]) - - proc = subprocess.Popen([args.apache_ctl, "-d", args.server_root, "-f", - args.config_file, "-t", "-D", "DUMP_VHOSTS"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - with open(os.path.join(tempdir, "vhosts"), "w") as vhosts_fd: - vhosts_fd.write(proc.communicate()[0]) - - return tempdir - - -def verify_config(args): - """Verifies server_root and config_file specify a valid config - - :param argparse.Namespace args: Parsed command line arguments - - """ - with open(os.devnull, "w") as devnull: - try: - subprocess.check_call([args.apache_ctl, "-d", args.server_root, - "-f", args.config_file, "-t"], - stdout=devnull, stderr=subprocess.STDOUT) - except OSError: - sys.exit(_NO_APACHECTL) - except subprocess.CalledProcessError: - sys.exit("Syntax check from apachectl failed") - - -def locate_config(apache_ctl): - """Uses the apachectl binary to find configuration files - - :param str apache_ctl: Path to `apachectl` binary - - - :returns: Path to Apache server root and main configuration file - :rtype: `tuple` of `str` - - """ - try: - proc = subprocess.Popen([apache_ctl, "-V"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - output, _ = proc.communicate() - except OSError: - sys.exit(_NO_APACHECTL) - - server_root = config_file = "" - for line in output.splitlines(): - # Relevant output lines are of the form: -D DIRECTIVE="VALUE" - if "HTTPD_ROOT" in line: - server_root = line[line.find('"') + 1:-1] - elif "SERVER_CONFIG_FILE" in line: - config_file = line[line.find('"') + 1:-1] - - if not (server_root and config_file): - sys.exit("Unable to locate Apache configuration. Please run this " - "script again and specify --server-root and --config-file") - - return server_root, config_file - - -def get_args(): - """Parses command line arguments - - :returns: Parsed command line options - :rtype: argparse.Namespace - - """ - parser = argparse.ArgumentParser(description=_DESCRIPTION) - parser.add_argument("-c", "--apache-ctl", default="apachectl", - help="path to the `apachectl` binary") - parser.add_argument("-d", "--server-root", - help=("location of the root directory of your Apache " - "configuration")) - parser.add_argument("-f", "--config-file", - help=("location of your main Apache configuration " - "file relative to the server root")) - args = parser.parse_args() - - # args.server_root XOR args.config_file - if bool(args.server_root) != bool(args.config_file): - sys.exit("If either --server-root and --config-file are specified, " - "they both must be included") - elif args.server_root and args.config_file: - args.server_root = os.path.abspath(args.server_root) - args.config_file = os.path.abspath(args.config_file) - - if args.config_file.startswith(args.server_root): - args.config_file = args.config_file[len(args.server_root) + 1:] - else: - sys.exit("This script expects the Apache configuration file to be " - "inside the server root") - - return args - - -def main(): - """Main script execution""" - args = get_args() - if args.server_root is None: - args.server_root, args.config_file = locate_config(args.apache_ctl) - - verify_config(args) - tempdir = setup_tempdir(args) - atexit.register(lambda: shutil.rmtree(tempdir)) - make_and_verify_selection(args.server_root, tempdir) - - tarpath = os.path.join(tempdir, "config.tar.gz") - with tarfile.open(tarpath, mode="w:gz") as tar: - tar.add(tempdir, arcname=".") - - # TODO: Submit tarpath - - -if __name__ == "__main__": - main() # pragma: no cover diff --git a/letshelp-certbot/letshelp_certbot/apache_test.py b/letshelp-certbot/letshelp_certbot/apache_test.py deleted file mode 100644 index 0853046b4..000000000 --- a/letshelp-certbot/letshelp_certbot/apache_test.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Tests for letshelp.letshelp_certbot_apache.py""" -import argparse -import functools -import os -import subprocess -import tarfile -import tempfile -import unittest - -# six is used in mock.patch() -import mock -import pkg_resources -import six # pylint: disable=unused-import - -import letshelp_certbot.apache as letshelp_le_apache - -_PARTIAL_CONF_PATH = os.path.join("mods-available", "ssl.load") -_PARTIAL_LINK_PATH = os.path.join("mods-enabled", "ssl.load") -_CONFIG_FILE = pkg_resources.resource_filename( - __name__, os.path.join("testdata", _PARTIAL_CONF_PATH)) -_PASSWD_FILE = pkg_resources.resource_filename( - __name__, os.path.join("testdata", "uncommonly_named_p4sswd")) -_KEY_FILE = pkg_resources.resource_filename( - __name__, os.path.join("testdata", "uncommonly_named_k3y")) -_SECRET_FILE = pkg_resources.resource_filename( - __name__, os.path.join("testdata", "super_secret_file.txt")) - - -_MODULE_NAME = "letshelp_certbot.apache" - - -_COMPILE_SETTINGS = """Server version: Apache/2.4.10 (Debian) -Server built: Mar 15 2015 09:51:43 -Server's Module Magic Number: 20120211:37 -Server loaded: APR 1.5.1, APR-UTIL 1.5.4 -Compiled using: APR 1.5.1, APR-UTIL 1.5.4 -Architecture: 64-bit -Server MPM: event - threaded: yes (fixed thread count) - forked: yes (variable process count) -Server compiled with.... - -D APR_HAS_SENDFILE - -D APR_HAS_MMAP - -D APR_HAVE_IPV6 (IPv4-mapped addresses enabled) - -D APR_USE_SYSVSEM_SERIALIZE - -D APR_USE_PTHREAD_SERIALIZE - -D SINGLE_LISTEN_UNSERIALIZED_ACCEPT - -D APR_HAS_OTHER_CHILD - -D AP_HAVE_RELIABLE_PIPED_LOGS - -D DYNAMIC_MODULE_LIMIT=256 - -D HTTPD_ROOT="/etc/apache2" - -D SUEXEC_BIN="/usr/lib/apache2/suexec" - -D DEFAULT_PIDLOG="/var/run/apache2.pid" - -D DEFAULT_SCOREBOARD="logs/apache_runtime_status" - -D DEFAULT_ERRORLOG="logs/error_log" - -D AP_TYPES_CONFIG_FILE="mime.types" - -D SERVER_CONFIG_FILE="apache2.conf" - -""" - - -class LetsHelpApacheTest(unittest.TestCase): - @mock.patch(_MODULE_NAME + ".copy_config") - def test_make_and_verify_selection(self, mock_copy_config): - mock_copy_config.return_value = (["apache2.conf"], ["apache2"]) - - with mock.patch("six.moves.input") as mock_input: - with mock.patch(_MODULE_NAME + ".sys.stdout"): - mock_input.side_effect = ["Yes", "No"] - letshelp_le_apache.make_and_verify_selection("root", "temp") - self.assertRaises( - SystemExit, letshelp_le_apache.make_and_verify_selection, - "server_root", "temp_dir") - - def test_copy_config(self): - tempdir = tempfile.mkdtemp() - server_root = pkg_resources.resource_filename(__name__, "testdata") - letshelp_le_apache.copy_config(server_root, tempdir) - - temp_testdata = os.path.join(tempdir, "testdata") - self.assertFalse(os.path.exists(os.path.join( - temp_testdata, os.path.basename(_PASSWD_FILE)))) - self.assertFalse(os.path.exists(os.path.join( - temp_testdata, os.path.basename(_KEY_FILE)))) - self.assertFalse(os.path.exists(os.path.join( - temp_testdata, os.path.basename(_SECRET_FILE)))) - self.assertTrue(os.path.exists(os.path.join( - temp_testdata, _PARTIAL_CONF_PATH))) - self.assertTrue(os.path.exists(os.path.join( - temp_testdata, _PARTIAL_LINK_PATH))) - - def test_copy_file_without_comments(self): - dest = tempfile.mkstemp()[1] - letshelp_le_apache.copy_file_without_comments(_PASSWD_FILE, dest) - - with open(_PASSWD_FILE) as original: - with open(dest) as copy: - for original_line, copied_line in zip(original, copy): - self.assertEqual(original_line, copied_line) - - @mock.patch(_MODULE_NAME + ".subprocess.Popen") - def test_safe_config_file(self, mock_popen): - mock_popen().communicate.return_value = ("PEM RSA private key", None) - self.assertFalse(letshelp_le_apache.safe_config_file("filename")) - - mock_popen().communicate.return_value = ("ASCII text", None) - self.assertFalse(letshelp_le_apache.safe_config_file(_PASSWD_FILE)) - self.assertFalse(letshelp_le_apache.safe_config_file(_KEY_FILE)) - self.assertFalse(letshelp_le_apache.safe_config_file(_SECRET_FILE)) - self.assertTrue(letshelp_le_apache.safe_config_file(_CONFIG_FILE)) - - @mock.patch(_MODULE_NAME + ".subprocess.Popen") - def test_tempdir(self, mock_popen): - mock_popen().communicate.side_effect = [ - ("version", None), ("modules", None), ("vhosts", None)] - args = _get_args() - - tempdir = letshelp_le_apache.setup_tempdir(args) - - with open(os.path.join(tempdir, "config_file")) as config_fd: - self.assertEqual(config_fd.read(), args.config_file + "\n") - - with open(os.path.join(tempdir, "version")) as version_fd: - self.assertEqual(version_fd.read(), "version") - - with open(os.path.join(tempdir, "modules")) as modules_fd: - self.assertEqual(modules_fd.read(), "modules") - - with open(os.path.join(tempdir, "vhosts")) as vhosts_fd: - self.assertEqual(vhosts_fd.read(), "vhosts") - - @mock.patch(_MODULE_NAME + ".subprocess.check_call") - def test_verify_config(self, mock_check_call): - args = _get_args() - mock_check_call.side_effect = [ - None, OSError, subprocess.CalledProcessError(1, "apachectl")] - - letshelp_le_apache.verify_config(args) - self.assertRaises(SystemExit, letshelp_le_apache.verify_config, args) - self.assertRaises(SystemExit, letshelp_le_apache.verify_config, args) - - @mock.patch(_MODULE_NAME + ".subprocess.Popen") - def test_locate_config(self, mock_popen): - mock_popen().communicate.side_effect = [ - OSError, ("bad_output", None), (_COMPILE_SETTINGS, None)] - - self.assertRaises( - SystemExit, letshelp_le_apache.locate_config, "ctl") - self.assertRaises( - SystemExit, letshelp_le_apache.locate_config, "ctl") - server_root, config_file = letshelp_le_apache.locate_config("ctl") - self.assertEqual(server_root, "/etc/apache2") - self.assertEqual(config_file, "apache2.conf") - - @mock.patch(_MODULE_NAME + ".argparse") - def test_get_args(self, mock_argparse): - argv = ["-d", "/etc/apache2"] - mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv) - self.assertRaises(SystemExit, letshelp_le_apache.get_args) - - server_root = "/etc/apache2" - config_file = server_root + "/apache2.conf" - argv = ["-d", server_root, "-f", config_file] - mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv) - args = letshelp_le_apache.get_args() - self.assertEqual(args.apache_ctl, "apachectl") - self.assertEqual(args.server_root, server_root) - self.assertEqual(args.config_file, os.path.basename(config_file)) - - server_root = "/etc/apache2" - config_file = "/etc/httpd/httpd.conf" - argv = ["-d", server_root, "-f", config_file] - mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv) - self.assertRaises(SystemExit, letshelp_le_apache.get_args) - - def test_main_with_args(self): - with mock.patch(_MODULE_NAME + ".get_args"): - self._test_main_common() - - def test_main_without_args(self): - with mock.patch(_MODULE_NAME + ".get_args") as get_args: - args = _get_args() - server_root, config_file = args.server_root, args.config_file - args.server_root = args.config_file = None - get_args.return_value = args - with mock.patch(_MODULE_NAME + ".locate_config") as locate: - locate.return_value = (server_root, config_file) - self._test_main_common() - - def _test_main_common(self): - with mock.patch(_MODULE_NAME + ".verify_config"): - with mock.patch(_MODULE_NAME + ".setup_tempdir") as mock_setup: - tempdir_path = tempfile.mkdtemp() - mock_setup.return_value = tempdir_path - with mock.patch(_MODULE_NAME + ".make_and_verify_selection"): - testdir_basename = "test" - os.mkdir(os.path.join(tempdir_path, testdir_basename)) - - letshelp_le_apache.main() - - tar = tarfile.open(os.path.join( - tempdir_path, "config.tar.gz")) - - tempdir = tar.next() - if tempdir is None: - self.fail("Invalid tarball!") # pragma: no cover - else: - self.assertTrue(tempdir.isdir()) - self.assertEqual(tempdir.name, ".") - - testdir = tar.next() - if testdir is None: - self.fail("Invalid tarball!") # pragma: no cover - else: - self.assertTrue(testdir.isdir()) - self.assertEqual(os.path.basename(testdir.name), - testdir_basename) - - self.assertEqual(tar.next(), None) - - -def _create_mock_parser(argv): - parser = argparse.ArgumentParser() - mock_parser = mock.MagicMock() - mock_parser.add_argument = parser.add_argument - mock_parser.parse_args = functools.partial(parser.parse_args, argv) - - return mock_parser - - -def _get_args(): - args = argparse.Namespace() - args.apache_ctl = "apache_ctl" - args.config_file = "config_file" - args.server_root = "server_root" - - return args - - -if __name__ == "__main__": - unittest.main() # pragma: no cover diff --git a/letshelp-certbot/letshelp_certbot/magic_typing.py b/letshelp-certbot/letshelp_certbot/magic_typing.py deleted file mode 100644 index d6b1ff056..000000000 --- a/letshelp-certbot/letshelp_certbot/magic_typing.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Shim class to not have to depend on typing module in prod.""" -import sys - - -class TypingClass(object): - """Ignore import errors by getting anything""" - def __getattr__(self, name): - return None - -try: - # mypy doesn't respect modifying sys.modules - from typing import * # pylint: disable=wildcard-import, unused-wildcard-import - from typing import Collection, IO # type: ignore - # pylint: enable=unused-import -except ImportError: - sys.modules[__name__] = TypingClass() diff --git a/letshelp-certbot/letshelp_certbot/magic_typing_test.py b/letshelp-certbot/letshelp_certbot/magic_typing_test.py deleted file mode 100644 index 41bc245fa..000000000 --- a/letshelp-certbot/letshelp_certbot/magic_typing_test.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Tests for letshelp_certbot.magic_typing.""" -import sys -import unittest - -import mock - - -class MagicTypingTest(unittest.TestCase): - """Tests for letshelp_certbot.magic_typing.""" - def test_import_success(self): - try: - import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover - typing_class_mock = mock.MagicMock() - text_mock = mock.MagicMock() - typing_class_mock.Text = text_mock - sys.modules['typing'] = typing_class_mock - if 'letshelp_certbot.magic_typing' in sys.modules: - del sys.modules['letshelp_certbot.magic_typing'] # pragma: no cover - from letshelp_certbot.magic_typing import Text - self.assertEqual(Text, text_mock) - del sys.modules['letshelp_certbot.magic_typing'] - sys.modules['typing'] = temp_typing - - def test_import_failure(self): - try: - import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover - sys.modules['typing'] = None - if 'letshelp_certbot.magic_typing' in sys.modules: - del sys.modules['letshelp_certbot.magic_typing'] # pragma: no cover - from letshelp_certbot.magic_typing import Text - self.assertTrue(Text is None) - del sys.modules['letshelp_certbot.magic_typing'] - sys.modules['typing'] = temp_typing - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/letshelp-certbot/letshelp_certbot/testdata/mods-available/ssl.load b/letshelp-certbot/letshelp_certbot/testdata/mods-available/ssl.load deleted file mode 100644 index 3d2336ae0..000000000 --- a/letshelp-certbot/letshelp_certbot/testdata/mods-available/ssl.load +++ /dev/null @@ -1,2 +0,0 @@ -# Depends: setenvif mime socache_shmcb -LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so diff --git a/letshelp-certbot/letshelp_certbot/testdata/mods-enabled/ssl.load b/letshelp-certbot/letshelp_certbot/testdata/mods-enabled/ssl.load deleted file mode 120000 index 9d7972384..000000000 --- a/letshelp-certbot/letshelp_certbot/testdata/mods-enabled/ssl.load +++ /dev/null @@ -1 +0,0 @@ -../mods-available/ssl.load \ No newline at end of file diff --git a/letshelp-certbot/letshelp_certbot/testdata/super_secret_file.txt b/letshelp-certbot/letshelp_certbot/testdata/super_secret_file.txt deleted file mode 100644 index 9f592eb7d..000000000 --- a/letshelp-certbot/letshelp_certbot/testdata/super_secret_file.txt +++ /dev/null @@ -1 +0,0 @@ -hunter2 diff --git a/letshelp-certbot/letshelp_certbot/testdata/uncommonly_named_k3y b/letshelp-certbot/letshelp_certbot/testdata/uncommonly_named_k3y deleted file mode 100644 index 659274d1d..000000000 --- a/letshelp-certbot/letshelp_certbot/testdata/uncommonly_named_k3y +++ /dev/null @@ -1,6 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIGrAgEAAiEAm2Fylv+Uz7trgTW8EBHP3FQSMeZs2GNQ6VRo1sIVJEkCAwEAAQIh -AJT0BA/xD01dFCAXzSNyj9nfSZa3NpqzJZZn/eOm7vghAhEAzUVNZn4lLLBD1R6N -E8TKNQIRAMHHyn3O5JeY36lwKwkUlEUCEAliRauN0L0+QZuYjfJ9aJECEGx4dru3 -rTPCyighdqWNlHUCEQCiLjlwSRtWgmMBudCkVjzt ------END RSA PRIVATE KEY----- diff --git a/letshelp-certbot/letshelp_certbot/testdata/uncommonly_named_p4sswd b/letshelp-certbot/letshelp_certbot/testdata/uncommonly_named_p4sswd deleted file mode 100644 index 3559c1d1f..000000000 --- a/letshelp-certbot/letshelp_certbot/testdata/uncommonly_named_p4sswd +++ /dev/null @@ -1 +0,0 @@ -johntheripper:$apr1$fIGE9.JL$jTCwNWZy9Ak/yvOLuOyzQ1 diff --git a/letshelp-certbot/readthedocs.org.requirements.txt b/letshelp-certbot/readthedocs.org.requirements.txt deleted file mode 100644 index b24681caa..000000000 --- a/letshelp-certbot/readthedocs.org.requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -# readthedocs.org gives no way to change the install command to "pip -# install -e certbot[docs]" (that would in turn install documentation -# dependencies), but it allows to specify a requirements.txt file at -# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) - -# Although ReadTheDocs certainly doesn't need to install the project -# in --editable mode (-e), just "pip install .[docs]" does not work as -# expected and "pip install -e certbot[docs]" must be used instead - --e letshelp-certbot[docs] diff --git a/letshelp-certbot/setup.cfg b/letshelp-certbot/setup.cfg deleted file mode 100644 index 2a9acf13d..000000000 --- a/letshelp-certbot/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal = 1 diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py deleted file mode 100644 index 448c145ce..000000000 --- a/letshelp-certbot/setup.py +++ /dev/null @@ -1,58 +0,0 @@ -from setuptools import find_packages -from setuptools import setup - -version = '0.7.0.dev0' - -install_requires = [ - 'mock', - 'setuptools', # pkg_resources -] - -docs_extras = [ - 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags - 'sphinx_rtd_theme', -] - -setup( - name='letshelp-certbot', - version=version, - description="Let's help Certbot client", - url='https://github.com/letsencrypt/letsencrypt', - author="Certbot Project", - author_email='client-dev@letsencrypt.org', - license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Security', - 'Topic :: System :: Installation/Setup', - 'Topic :: System :: Networking', - 'Topic :: System :: Systems Administration', - 'Topic :: Utilities', - ], - - packages=find_packages(), - include_package_data=True, - install_requires=install_requires, - extras_require={ - 'docs': docs_extras, - }, - entry_points={ - 'console_scripts': [ - 'letshelp-certbot-apache = letshelp_certbot.apache:main', - ], - }, - test_suite='letshelp_certbot', -) diff --git a/linter_plugin.py b/linter_plugin.py index 1754b1a2a..b6388e2c7 100644 --- a/linter_plugin.py +++ b/linter_plugin.py @@ -10,7 +10,7 @@ from pylint.checkers import BaseChecker from pylint.interfaces import IAstroidChecker # Modules in theses packages can import the os module. -WHITELIST_PACKAGES = ['acme', 'certbot_compatibility_test', 'letshelp_certbot', 'lock_test'] +WHITELIST_PACKAGES = ['acme', 'certbot_compatibility_test', 'lock_test'] class ForbidStandardOsModule(BaseChecker): diff --git a/mypy.ini b/mypy.ini index 188ed031f..a19fa2a5f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,3 @@ python_version = 2.7 [mypy-acme.magic_typing_test] ignore_errors = True - -[mypy-letshelp_certbot.magic_typing_test] -ignore_errors = True diff --git a/tools/_venv_common.py b/tools/_venv_common.py index c61385054..75d0d5d33 100644 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -39,7 +39,6 @@ REQUIREMENTS = [ '-e certbot-dns-route53', '-e certbot-dns-sakuracloud', '-e certbot-nginx', - '-e letshelp-certbot', '-e certbot-compatibility-test', '-e certbot-ci', ] diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 192708957..0b47fa5f8 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -12,7 +12,7 @@ import re import subprocess import sys -SKIP_PROJECTS_ON_WINDOWS = ['certbot-apache', 'letshelp-certbot'] +SKIP_PROJECTS_ON_WINDOWS = ['certbot-apache'] def call_with_print(command): diff --git a/tox.cover.py b/tox.cover.py index 0ef5c0d07..3e69a14d6 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -9,7 +9,7 @@ DEFAULT_PACKAGES = [ 'certbot_dns_digitalocean', 'certbot_dns_dnsimple', 'certbot_dns_dnsmadeeasy', 'certbot_dns_gehirn', 'certbot_dns_google', 'certbot_dns_linode', 'certbot_dns_luadns', 'certbot_dns_nsone', 'certbot_dns_ovh', 'certbot_dns_rfc2136', 'certbot_dns_route53', - 'certbot_dns_sakuracloud', 'certbot_nginx', 'letshelp_certbot'] + 'certbot_dns_sakuracloud', 'certbot_nginx'] COVER_THRESHOLDS = { 'certbot': {'linux': 96, 'windows': 96}, @@ -30,10 +30,9 @@ COVER_THRESHOLDS = { 'certbot_dns_route53': {'linux': 92, 'windows': 92}, 'certbot_dns_sakuracloud': {'linux': 97, 'windows': 97}, 'certbot_nginx': {'linux': 97, 'windows': 97}, - 'letshelp_certbot': {'linux': 100, 'windows': 100} } -SKIP_PROJECTS_ON_WINDOWS = ['certbot-apache', 'letshelp-certbot'] +SKIP_PROJECTS_ON_WINDOWS = ['certbot-apache'] def cover(package): diff --git a/tox.ini b/tox.ini index b2710ce35..3903cdf45 100644 --- a/tox.ini +++ b/tox.ini @@ -34,8 +34,7 @@ all_packages = certbot[dev] \ certbot-apache \ {[base]dns_packages} \ - certbot-nginx \ - letshelp-certbot + certbot-nginx install_packages = python {toxinidir}/tools/pip_install_editable.py {[base]all_packages} source_paths = @@ -58,7 +57,6 @@ source_paths = certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx - letshelp-certbot/letshelp_certbot tests/lock_test.py [testenv] From 99b1538d0a70986c7e925e8c091b2f36bded9a4b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 18 Feb 2020 11:55:48 -0800 Subject: [PATCH 12/92] Fix spurious pylint errors. (#7780) This fixes (part of) the problem identified in https://github.com/certbot/certbot/pull/7657#issuecomment-586506340. When I tested our pylint setup on Python 3.5.9, 3.6.9, or 3.6.10, tests failed with: ``` ************* Module acme.challenges acme/acme/challenges.py:57:15: E1101: Instance of 'UnrecognizedChallenge' has no 'jobj' member (no-member) ************* Module acme.jws acme/acme/jws.py:28:16: E1101: Class 'Signature' has no '_orig_slots' member (no-member) ``` These errors did not occur for me on Python 3.6.7 or Python 3.7+. You also cannot run our lint setup on Python 2.7 because our pinned version of pylint's dependency `asteroid` does not support Python 2. Because of this, `pylint` is not installed in the virtual environment created by `tools/venv.py` and our [`lint` environment in tox specifies that Python 3 should be used](https://github.com/certbot/certbot/blob/fd64c8c33b2176e6569d64d30776bd5fc9fd3820/tox.ini#L132). I tried updating pylint and its dependencies to fix the problem, but they still occur so I think adding back these disable checks on these lines again is the best fix for now. --- acme/acme/challenges.py | 2 +- acme/acme/jws.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index f3fb19b42..39c8d6269 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -54,7 +54,7 @@ class UnrecognizedChallenge(Challenge): object.__setattr__(self, "jobj", jobj) def to_partial_json(self): - return self.jobj + return self.jobj # pylint: disable=no-member @classmethod def from_json(cls, jobj): diff --git a/acme/acme/jws.py b/acme/acme/jws.py index 9128f56b3..2188c3727 100644 --- a/acme/acme/jws.py +++ b/acme/acme/jws.py @@ -25,7 +25,7 @@ class Header(jose.Header): class Signature(jose.Signature): """ACME-specific Signature. Uses ACME-specific Header for customer fields.""" - __slots__ = jose.Signature._orig_slots + __slots__ = jose.Signature._orig_slots # pylint: disable=no-member # TODO: decoder/encoder should accept cls? Otherwise, subclassing # JSONObjectWithFields is tricky... From 42dda355c598d9e1938f6ab4eb30f4fc05b5964f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 18 Feb 2020 14:54:07 -0800 Subject: [PATCH 13/92] Correct AutoHSTS docs (#7767) domains is a list of strings, not a single string. * Correct AutoHSTS docs. * Fix Apache enable_autohsts docs. --- certbot-apache/certbot_apache/_internal/configurator.py | 2 +- certbot/certbot/plugins/enhancements.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 465237590..8daa28173 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -2471,7 +2471,7 @@ class ApacheConfigurator(common.Installer): :type _unused_lineage: certbot._internal.storage.RenewableCert :param domains: List of domains in certificate to enhance - :type domains: str + :type domains: `list` of `str` """ self._autohsts_fetch_state() diff --git a/certbot/certbot/plugins/enhancements.py b/certbot/certbot/plugins/enhancements.py index be9b7933d..4abce2d2f 100644 --- a/certbot/certbot/plugins/enhancements.py +++ b/certbot/certbot/plugins/enhancements.py @@ -153,7 +153,7 @@ class AutoHSTSEnhancement(object): :type lineage: certbot.interfaces.RenewableCert :param domains: List of domains in certificate to enhance - :type domains: str + :type domains: `list` of `str` """ # This is used to configure internal new style enhancements in Certbot. These From c883efde0f649a0a7f5da1c35cbb1ae54d752109 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 20 Feb 2020 14:35:47 -0800 Subject: [PATCH 14/92] add pgp key docs (#7765) Fixes #7613. --- certbot/docs/packaging.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/certbot/docs/packaging.rst b/certbot/docs/packaging.rst index d2a94dbe7..06ae7bbff 100644 --- a/certbot/docs/packaging.rst +++ b/certbot/docs/packaging.rst @@ -33,6 +33,10 @@ example: `v0.11.1`. .. _`Semantic Versioning`: http://semver.org/ +Our packages are cryptographically signed and their signature can be verified +using the PGP key ``A2CFB51FA275A7286234E7B24D17C995CD9775F2``. This key can be +found on major key servers and at https://dl.eff.org/certbot.pub. + Notes for package maintainers ============================= From 7d79c91e9b52ef75d7ac86232dbbedc84f066587 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 21 Feb 2020 11:18:53 -0800 Subject: [PATCH 15/92] Move our macOS tests to Azure Pipelines (#7793) [Our macOS tests are failing](https://travis-ci.com/certbot/certbot/builds/149965318) again this time due to the problem described at https://travis-ci.community/t/macos-build-fails-because-of-homebrew-bundle-unknown-command/7296/14. I tried adding `update: true` to the Homebrew config as described in that thread, but [it didn't work](https://travis-ci.com/certbot/certbot/builds/150070374). I also tried updating the macOS image we use which [didn't work](https://travis-ci.com/certbot/certbot/builds/150072389). Since we continue to have problems with macOS on Travis, let try moving the tests to Azure Pipelines. * test macos * Remove Travis macOS setup * add displayName --- .azure-pipelines/templates/tests-suite.yml | 24 +++++++++++++++++----- .travis.yml | 19 ----------------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/.azure-pipelines/templates/tests-suite.yml b/.azure-pipelines/templates/tests-suite.yml index 119f755a6..069ea94d6 100644 --- a/.azure-pipelines/templates/tests-suite.yml +++ b/.azure-pipelines/templates/tests-suite.yml @@ -1,22 +1,36 @@ jobs: - job: test - pool: - vmImage: vs2017-win2016 strategy: matrix: - py35: + macos-py27: + IMAGE_NAME: macOS-10.14 + PYTHON_VERSION: 2.7 + TOXENV: py27 + macos-py38: + IMAGE_NAME: macOS-10.14 + PYTHON_VERSION: 3.8 + TOXENV: py38 + windows-py35: + IMAGE_NAME: vs2017-win2016 PYTHON_VERSION: 3.5 TOXENV: py35 - py37-cover: + windows-py37-cover: + IMAGE_NAME: vs2017-win2016 PYTHON_VERSION: 3.7 TOXENV: py37-cover - integration-certbot: + windows-integration-certbot: + IMAGE_NAME: vs2017-win2016 PYTHON_VERSION: 3.7 TOXENV: integration-certbot PYTEST_ADDOPTS: --numprocesses 4 + pool: + vmImage: $(IMAGE_NAME) variables: - group: certbot-common steps: + - bash: brew install augeas + condition: startswith(variables['IMAGE_NAME'], 'macOS') + displayName: Install Augeas - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) diff --git a/.travis.yml b/.travis.yml index 1eae66333..e5354898d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ cache: - $HOME/.cache/pip before_script: - - 'if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ulimit -n 1024 ; fi' # On Travis, the fastest parallelization for integration tests has proved to be 4. - 'if [[ "$TOXENV" == *"integration"* ]]; then export PYTEST_ADDOPTS="--numprocesses 4"; fi' # Use Travis retry feature for farm tests since they are flaky @@ -224,24 +223,6 @@ matrix: packages: # don't install nginx and apache - libaugeas0 <<: *extended-test-suite - - language: generic - env: TOXENV=py27 - os: osx - addons: - homebrew: - packages: - - augeas - - python2 - <<: *extended-test-suite - - language: generic - env: TOXENV=py3 - os: osx - addons: - homebrew: - packages: - - augeas - - python3 - <<: *extended-test-suite # container-based infrastructure sudo: false From 84b57fac9341453b12135cdf26d9ede092e2c3aa Mon Sep 17 00:00:00 2001 From: Raklyon <46962656+Raklyon@users.noreply.github.com> Date: Fri, 21 Feb 2020 21:30:58 +0100 Subject: [PATCH 16/92] Refactor cli.py, splitting in it smaller submodules (#6803) * Refactor cli.py into a package with submodules * Added unit tests for helpful module in cli. * Fixed linter errors * Fixed pylint issues * Updated changelog.md * Fixed test failing and mypy error. Appeared a new pylint error (seems to be in conflict with mypy) mypy require zope.interface to be imported but when imported it is not used and pylint throws an error. * Fixed pylint errors * Apply changes to cli since last merge from master (efc8d49806b14a31d88cfc0f1b6daca1dd373d8d) * Fix lint * Remaining lint errors Co-authored-by: Adrien Ferrand --- certbot/CHANGELOG.md | 2 +- certbot/certbot/_internal/cli.py | 1581 ----------------- certbot/certbot/_internal/cli/__init__.py | 526 ++++++ .../certbot/_internal/cli/cli_constants.py | 107 ++ certbot/certbot/_internal/cli/cli_utils.py | 239 +++ certbot/certbot/_internal/cli/group_adder.py | 19 + certbot/certbot/_internal/cli/helpful.py | 468 +++++ certbot/certbot/_internal/cli/paths_parser.py | 50 + .../certbot/_internal/cli/plugins_parsing.py | 97 + .../cli/report_config_interaction.py | 27 + certbot/certbot/_internal/cli/subparsers.py | 72 + certbot/certbot/_internal/cli/verb_help.py | 106 ++ certbot/tests/cli_test.py | 2 +- certbot/tests/helpful_test.py | 193 ++ 14 files changed, 1906 insertions(+), 1583 deletions(-) delete mode 100644 certbot/certbot/_internal/cli.py create mode 100644 certbot/certbot/_internal/cli/__init__.py create mode 100644 certbot/certbot/_internal/cli/cli_constants.py create mode 100644 certbot/certbot/_internal/cli/cli_utils.py create mode 100644 certbot/certbot/_internal/cli/group_adder.py create mode 100644 certbot/certbot/_internal/cli/helpful.py create mode 100644 certbot/certbot/_internal/cli/paths_parser.py create mode 100644 certbot/certbot/_internal/cli/plugins_parsing.py create mode 100644 certbot/certbot/_internal/cli/report_config_interaction.py create mode 100644 certbot/certbot/_internal/cli/subparsers.py create mode 100644 certbot/certbot/_internal/cli/verb_help.py create mode 100644 certbot/tests/helpful_test.py diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index ff3061e01..126b07eec 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -13,7 +13,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* +* certbot._internal.cli is now a package split in submodules instead of a whole module. ### Fixed diff --git a/certbot/certbot/_internal/cli.py b/certbot/certbot/_internal/cli.py deleted file mode 100644 index c0dbf7424..000000000 --- a/certbot/certbot/_internal/cli.py +++ /dev/null @@ -1,1581 +0,0 @@ -"""Certbot command line argument & config processing.""" -# pylint: disable=too-many-lines -from __future__ import print_function - -import argparse -import copy -import glob -import logging.handlers -import sys - -import configargparse -import six -import zope.component -import zope.interface -from zope.interface import interfaces as zope_interfaces - -from acme import challenges -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import Optional -import certbot -from certbot import crypto_util -from certbot import errors -from certbot import interfaces -from certbot import util -from certbot._internal import constants -from certbot._internal import hooks -from certbot._internal.plugins import disco as plugins_disco -import certbot._internal.plugins.selection as plugin_selection -from certbot.compat import os -from certbot.display import util as display_util -import certbot.plugins.enhancements as enhancements - -logger = logging.getLogger(__name__) - -# Global, to save us from a lot of argument passing within the scope of this module -helpful_parser = None # type: Optional[HelpfulArgumentParser] - -# For help strings, figure out how the user ran us. -# When invoked from letsencrypt-auto, sys.argv[0] is something like: -# "/home/user/.local/share/certbot/bin/certbot" -# Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before -# running letsencrypt-auto (and sudo stops us from seeing if they did), so it -# should only be used for purposes where inability to detect letsencrypt-auto -# fails safely - -LEAUTO = "letsencrypt-auto" -if "CERTBOT_AUTO" in os.environ: - # if we're here, this is probably going to be certbot-auto, unless the - # user saved the script under a different name - LEAUTO = os.path.basename(os.environ["CERTBOT_AUTO"]) - -old_path_fragment = os.path.join(".local", "share", "letsencrypt") -new_path_prefix = os.path.abspath(os.path.join(os.sep, "opt", - "eff.org", "certbot", "venv")) -if old_path_fragment in sys.argv[0] or sys.argv[0].startswith(new_path_prefix): - cli_command = LEAUTO -else: - cli_command = "certbot" - -# Argparse's help formatting has a lot of unhelpful peculiarities, so we want -# to replace as much of it as we can... - -# This is the stub to include in help generated by argparse -SHORT_USAGE = """ - {0} [SUBCOMMAND] [options] [-d DOMAIN] [-d DOMAIN] ... - -Certbot can obtain and install HTTPS/TLS/SSL certificates. By default, -it will attempt to use a webserver both for obtaining and installing the -certificate. """.format(cli_command) - -# This section is used for --help and --help all ; it needs information -# about installed plugins to be fully formatted -COMMAND_OVERVIEW = """The most common SUBCOMMANDS and flags are: - -obtain, install, and renew certificates: - (default) run Obtain & install a certificate in your current webserver - certonly Obtain or renew a certificate, but do not install it - renew Renew all previously obtained certificates that are near expiry - enhance Add security enhancements to your existing configuration - -d DOMAINS Comma-separated list of domains to obtain a certificate for - - %s - --standalone Run a standalone webserver for authentication - %s - --webroot Place files in a server's webroot folder for authentication - --manual Obtain certificates interactively, or using shell script hooks - - -n Run non-interactively - --test-cert Obtain a test certificate from a staging server - --dry-run Test "renew" or "certonly" without saving any certificates to disk - -manage certificates: - certificates Display information about certificates you have from Certbot - revoke Revoke a certificate (supply --cert-name or --cert-path) - delete Delete a certificate (supply --cert-name) - -manage your account: - register Create an ACME account - unregister Deactivate an ACME account - update_account Update an ACME account - --agree-tos Agree to the ACME server's Subscriber Agreement - -m EMAIL Email address for important account notifications -""" - -# This is the short help for certbot --help, where we disable argparse -# altogether -HELP_AND_VERSION_USAGE = """ -More detailed help: - - -h, --help [TOPIC] print this message, or detailed help on a topic; - the available TOPICS are: - - all, automation, commands, paths, security, testing, or any of the - subcommands or plugins (certonly, renew, install, register, nginx, - apache, standalone, webroot, etc.) - -h all print a detailed help page including all topics - --version print the version number -""" - - -# These argparse parameters should be removed when detecting defaults. -ARGPARSE_PARAMS_TO_REMOVE = ("const", "nargs", "type",) - - -# These sets are used when to help detect options set by the user. -EXIT_ACTIONS = set(("help", "version",)) - - -ZERO_ARG_ACTIONS = set(("store_const", "store_true", - "store_false", "append_const", "count",)) - - -# Maps a config option to a set of config options that may have modified it. -# This dictionary is used recursively, so if A modifies B and B modifies C, -# it is determined that C was modified by the user if A was modified. -VAR_MODIFIERS = {"account": set(("server",)), - "renew_hook": set(("deploy_hook",)), - "server": set(("dry_run", "staging",)), - "webroot_map": set(("webroot_path",))} - - -def report_config_interaction(modified, modifiers): - """Registers config option interaction to be checked by set_by_cli. - - This function can be called by during the __init__ or - add_parser_arguments methods of plugins to register interactions - between config options. - - :param modified: config options that can be modified by modifiers - :type modified: iterable or str (string_types) - :param modifiers: config options that modify modified - :type modifiers: iterable or str (string_types) - - """ - if isinstance(modified, six.string_types): - modified = (modified,) - if isinstance(modifiers, six.string_types): - modifiers = (modifiers,) - - for var in modified: - VAR_MODIFIERS.setdefault(var, set()).update(modifiers) - - -class _Default(object): - """A class to use as a default to detect if a value is set by a user""" - - def __bool__(self): - return False - - def __eq__(self, other): - return isinstance(other, _Default) - - def __hash__(self): - return id(_Default) - - def __nonzero__(self): - return self.__bool__() - - -def set_by_cli(var): - """ - Return True if a particular config variable has been set by the user - (CLI or config file) including if the user explicitly set it to the - default. Returns False if the variable was assigned a default value. - """ - detector = set_by_cli.detector # type: ignore - if detector is None and helpful_parser is not None: - # Setup on first run: `detector` is a weird version of config in which - # the default value of every attribute is wrangled to be boolean-false - plugins = plugins_disco.PluginsRegistry.find_all() - # reconstructed_args == sys.argv[1:], or whatever was passed to main() - reconstructed_args = helpful_parser.args + [helpful_parser.verb] - detector = set_by_cli.detector = prepare_and_parse_args( # type: ignore - plugins, reconstructed_args, detect_defaults=True) - # propagate plugin requests: eg --standalone modifies config.authenticator - detector.authenticator, detector.installer = ( # type: ignore - plugin_selection.cli_plugin_requests(detector)) - - if not isinstance(getattr(detector, var), _Default): - logger.debug("Var %s=%s (set by user).", var, getattr(detector, var)) - return True - - for modifier in VAR_MODIFIERS.get(var, []): - if set_by_cli(modifier): - logger.debug("Var %s=%s (set by user).", - var, VAR_MODIFIERS.get(var, [])) - return True - - return False - -# static housekeeping var -# functions attributed are not supported by mypy -# https://github.com/python/mypy/issues/2087 -set_by_cli.detector = None # type: ignore - - -def has_default_value(option, value): - """Does option have the default value? - - If the default value of option is not known, False is returned. - - :param str option: configuration variable being considered - :param value: value of the configuration variable named option - - :returns: True if option has the default value, otherwise, False - :rtype: bool - - """ - if helpful_parser is not None: - return (option in helpful_parser.defaults and - helpful_parser.defaults[option] == value) - return False - - -def option_was_set(option, value): - """Was option set by the user or does it differ from the default? - - :param str option: configuration variable being considered - :param value: value of the configuration variable named option - - :returns: True if the option was set, otherwise, False - :rtype: bool - - """ - return set_by_cli(option) or not has_default_value(option, value) - - -def argparse_type(variable): - """Return our argparse type function for a config variable (default: str)""" - # pylint: disable=protected-access - if helpful_parser is not None: - for action in helpful_parser.parser._actions: - if action.type is not None and action.dest == variable: - return action.type - return str - -def read_file(filename, mode="rb"): - """Returns the given file's contents. - - :param str filename: path to file - :param str mode: open mode (see `open`) - - :returns: absolute path of filename and its contents - :rtype: tuple - - :raises argparse.ArgumentTypeError: File does not exist or is not readable. - - """ - try: - filename = os.path.abspath(filename) - with open(filename, mode) as the_file: - contents = the_file.read() - return filename, contents - except IOError as exc: - raise argparse.ArgumentTypeError(exc.strerror) - - -def flag_default(name): - """Default value for CLI flag.""" - # XXX: this is an internal housekeeping notion of defaults before - # argparse has been set up; it is not accurate for all flags. Call it - # with caution. Plugin defaults are missing, and some things are using - # defaults defined in this file, not in constants.py :( - return copy.deepcopy(constants.CLI_DEFAULTS[name]) - - -def config_help(name, hidden=False): - """Extract the help message for an `.IConfig` attribute.""" - if hidden: - return argparse.SUPPRESS - field = interfaces.IConfig.__getitem__(name) # type: zope.interface.interface.Attribute - return field.__doc__ - - -class HelpfulArgumentGroup(object): - """Emulates an argparse group for use with HelpfulArgumentParser. - - This class is used in the add_group method of HelpfulArgumentParser. - Command line arguments can be added to the group, but help - suppression and default detection is applied by - HelpfulArgumentParser when necessary. - - """ - def __init__(self, helpful_arg_parser, topic): - self._parser = helpful_arg_parser - self._topic = topic - - def add_argument(self, *args, **kwargs): - """Add a new command line argument to the argument group.""" - self._parser.add(self._topic, *args, **kwargs) - -class CustomHelpFormatter(argparse.HelpFormatter): - """This is a clone of ArgumentDefaultsHelpFormatter, with bugfixes. - - In particular we fix https://bugs.python.org/issue28742 - """ - - def _get_help_string(self, action): - helpstr = action.help - if '%(default)' not in action.help and '(default:' not in action.help: - if action.default != argparse.SUPPRESS: - defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - helpstr += ' (default: %(default)s)' - return helpstr - -# The attributes here are: -# short: a string that will be displayed by "certbot -h commands" -# opts: a string that heads the section of flags with which this command is documented, -# both for "certbot -h SUBCOMMAND" and "certbot -h all" -# usage: an optional string that overrides the header of "certbot -h SUBCOMMAND" -VERB_HELP = [ - ("run (default)", { - "short": "Obtain/renew a certificate, and install it", - "opts": "Options for obtaining & installing certificates", - "usage": SHORT_USAGE.replace("[SUBCOMMAND]", ""), - "realname": "run" - }), - ("certonly", { - "short": "Obtain or renew a certificate, but do not install it", - "opts": "Options for modifying how a certificate is obtained", - "usage": ("\n\n certbot certonly [options] [-d DOMAIN] [-d DOMAIN] ...\n\n" - "This command obtains a TLS/SSL certificate without installing it anywhere.") - }), - ("renew", { - "short": "Renew all certificates (or one specified with --cert-name)", - "opts": ("The 'renew' subcommand will attempt to renew all" - " certificates (or more precisely, certificate lineages) you have" - " previously obtained if they are close to expiry, and print a" - " summary of the results. By default, 'renew' will reuse the options" - " used to create obtain or most recently successfully renew each" - " certificate lineage. You can try it with `--dry-run` first. For" - " more fine-grained control, you can renew individual lineages with" - " the `certonly` subcommand. Hooks are available to run commands" - " before and after renewal; see" - " https://certbot.eff.org/docs/using.html#renewal for more" - " information on these."), - "usage": "\n\n certbot renew [--cert-name CERTNAME] [options]\n\n" - }), - ("certificates", { - "short": "List certificates managed by Certbot", - "opts": "List certificates managed by Certbot", - "usage": ("\n\n certbot certificates [options] ...\n\n" - "Print information about the status of certificates managed by Certbot.") - }), - ("delete", { - "short": "Clean up all files related to a certificate", - "opts": "Options for deleting a certificate", - "usage": "\n\n certbot delete --cert-name CERTNAME\n\n" - }), - ("revoke", { - "short": "Revoke a certificate specified with --cert-path or --cert-name", - "opts": "Options for revocation of certificates", - "usage": "\n\n certbot revoke [--cert-path /path/to/fullchain.pem | " - "--cert-name example.com] [options]\n\n" - }), - ("register", { - "short": "Register for account with Let's Encrypt / other ACME server", - "opts": "Options for account registration", - "usage": "\n\n certbot register --email user@example.com [options]\n\n" - }), - ("update_account", { - "short": "Update existing account with Let's Encrypt / other ACME server", - "opts": "Options for account modification", - "usage": "\n\n certbot update_account --email updated_email@example.com [options]\n\n" - }), - ("unregister", { - "short": "Irrevocably deactivate your account", - "opts": "Options for account deactivation.", - "usage": "\n\n certbot unregister [options]\n\n" - }), - ("install", { - "short": "Install an arbitrary certificate in a server", - "opts": "Options for modifying how a certificate is deployed", - "usage": "\n\n certbot install --cert-path /path/to/fullchain.pem " - " --key-path /path/to/private-key [options]\n\n" - }), - ("rollback", { - "short": "Roll back server conf changes made during certificate installation", - "opts": "Options for rolling back server configuration changes", - "usage": "\n\n certbot rollback --checkpoints 3 [options]\n\n" - }), - ("plugins", { - "short": "List plugins that are installed and available on your system", - "opts": 'Options for the "plugins" subcommand', - "usage": "\n\n certbot plugins [options]\n\n" - }), - ("update_symlinks", { - "short": "Recreate symlinks in your /etc/letsencrypt/live/ directory", - "opts": ("Recreates certificate and key symlinks in {0}, if you changed them by hand " - "or edited a renewal configuration file".format( - os.path.join(flag_default("config_dir"), "live"))), - "usage": "\n\n certbot update_symlinks [options]\n\n" - }), - ("enhance", { - "short": "Add security enhancements to your existing configuration", - "opts": ("Helps to harden the TLS configuration by adding security enhancements " - "to already existing configuration."), - "usage": "\n\n certbot enhance [options]\n\n" - }), - -] -# VERB_HELP is a list in order to preserve order, but a dict is sometimes useful -VERB_HELP_MAP = dict(VERB_HELP) - - -class HelpfulArgumentParser(object): - """Argparse Wrapper. - - This class wraps argparse, adding the ability to make --help less - verbose, and request help on specific subcategories at a time, eg - 'certbot --help security' for security options. - - """ - - - def __init__(self, args, plugins, detect_defaults=False): - from certbot._internal import main - self.VERBS = { - "auth": main.certonly, - "certonly": main.certonly, - "run": main.run, - "install": main.install, - "plugins": main.plugins_cmd, - "register": main.register, - "update_account": main.update_account, - "unregister": main.unregister, - "renew": main.renew, - "revoke": main.revoke, - "rollback": main.rollback, - "everything": main.run, - "update_symlinks": main.update_symlinks, - "certificates": main.certificates, - "delete": main.delete, - "enhance": main.enhance, - } - - # Get notification function for printing - try: - self.notify = zope.component.getUtility( - interfaces.IDisplay).notification - except zope_interfaces.ComponentLookupError: - self.notify = display_util.NoninteractiveDisplay( - sys.stdout).notification - - - # List of topics for which additional help can be provided - HELP_TOPICS = ["all", "security", "paths", "automation", "testing"] - HELP_TOPICS += list(self.VERBS) + self.COMMANDS_TOPICS + ["manage"] - - plugin_names = list(plugins) - self.help_topics = HELP_TOPICS + plugin_names + [None] # type: ignore - - self.detect_defaults = detect_defaults - self.args = args - - if self.args and self.args[0] == 'help': - self.args[0] = '--help' - - self.determine_verb() - help1 = self.prescan_for_flag("-h", self.help_topics) - help2 = self.prescan_for_flag("--help", self.help_topics) - if isinstance(help1, bool) and isinstance(help2, bool): - self.help_arg = help1 or help2 - else: - self.help_arg = help1 if isinstance(help1, six.string_types) else help2 - - short_usage = self._usage_string(plugins, self.help_arg) - - self.visible_topics = self.determine_help_topics(self.help_arg) - - # elements are added by .add_group() - self.groups = {} # type: Dict[str, argparse._ArgumentGroup] - # elements are added by .parse_args() - self.defaults = {} # type: Dict[str, Any] - - self.parser = configargparse.ArgParser( - prog="certbot", - usage=short_usage, - formatter_class=CustomHelpFormatter, - args_for_setting_config_path=["-c", "--config"], - default_config_files=flag_default("config_files"), - config_arg_help_message="path to config file (default: {0})".format( - " and ".join(flag_default("config_files")))) - - # This is the only way to turn off overly verbose config flag documentation - self.parser._add_config_file_help = False - - # Help that are synonyms for --help subcommands - COMMANDS_TOPICS = ["command", "commands", "subcommand", "subcommands", "verbs"] - def _list_subcommands(self): - longest = max(len(v) for v in VERB_HELP_MAP) - - text = "The full list of available SUBCOMMANDS is:\n\n" - for verb, props in sorted(VERB_HELP): - doc = props.get("short", "") - text += '{0:<{length}} {1}\n'.format(verb, doc, length=longest) - - text += "\nYou can get more help on a specific subcommand with --help SUBCOMMAND\n" - return text - - def _usage_string(self, plugins, help_arg): - """Make usage strings late so that plugins can be initialised late - - :param plugins: all discovered plugins - :param help_arg: False for none; True for --help; "TOPIC" for --help TOPIC - :rtype: str - :returns: a short usage string for the top of --help TOPIC) - """ - if "nginx" in plugins: - nginx_doc = "--nginx Use the Nginx plugin for authentication & installation" - else: - nginx_doc = "(the certbot nginx plugin is not installed)" - if "apache" in plugins: - apache_doc = "--apache Use the Apache plugin for authentication & installation" - else: - apache_doc = "(the certbot apache plugin is not installed)" - - usage = SHORT_USAGE - if help_arg is True: - self.notify(usage + COMMAND_OVERVIEW % (apache_doc, nginx_doc) + HELP_AND_VERSION_USAGE) - sys.exit(0) - elif help_arg in self.COMMANDS_TOPICS: - self.notify(usage + self._list_subcommands()) - sys.exit(0) - elif help_arg == "all": - # if we're doing --help all, the OVERVIEW is part of the SHORT_USAGE at - # the top; if we're doing --help someothertopic, it's OT so it's not - usage += COMMAND_OVERVIEW % (apache_doc, nginx_doc) - else: - custom = VERB_HELP_MAP.get(help_arg, {}).get("usage", None) - usage = custom if custom else usage - - return usage - - def remove_config_file_domains_for_renewal(self, parsed_args): - """Make "certbot renew" safe if domains are set in cli.ini.""" - # Works around https://github.com/certbot/certbot/issues/4096 - if self.verb == "renew": - for source, flags in self.parser._source_to_settings.items(): # pylint: disable=protected-access - if source.startswith("config_file") and "domains" in flags: - parsed_args.domains = _Default() if self.detect_defaults else [] - - def parse_args(self): - """Parses command line arguments and returns the result. - - :returns: parsed command line arguments - :rtype: argparse.Namespace - - """ - parsed_args = self.parser.parse_args(self.args) - parsed_args.func = self.VERBS[self.verb] - parsed_args.verb = self.verb - - self.remove_config_file_domains_for_renewal(parsed_args) - - if self.detect_defaults: - return parsed_args - - self.defaults = dict((key, copy.deepcopy(self.parser.get_default(key))) - for key in vars(parsed_args)) - - # Do any post-parsing homework here - - if self.verb == "renew": - if parsed_args.force_interactive: - raise errors.Error( - "{0} cannot be used with renew".format( - constants.FORCE_INTERACTIVE_FLAG)) - parsed_args.noninteractive_mode = True - - if parsed_args.force_interactive and parsed_args.noninteractive_mode: - raise errors.Error( - "Flag for non-interactive mode and {0} conflict".format( - constants.FORCE_INTERACTIVE_FLAG)) - - if parsed_args.staging or parsed_args.dry_run: - self.set_test_server(parsed_args) - - if parsed_args.csr: - self.handle_csr(parsed_args) - - if parsed_args.must_staple: - parsed_args.staple = True - - if parsed_args.validate_hooks: - hooks.validate_hooks(parsed_args) - - if parsed_args.allow_subset_of_names: - if any(util.is_wildcard_domain(d) for d in parsed_args.domains): - raise errors.Error("Using --allow-subset-of-names with a" - " wildcard domain is not supported.") - - if parsed_args.hsts and parsed_args.auto_hsts: - raise errors.Error( - "Parameters --hsts and --auto-hsts cannot be used simultaneously.") - - return parsed_args - - def set_test_server(self, parsed_args): - """We have --staging/--dry-run; perform sanity check and set config.server""" - - # Flag combinations should produce these results: - # | --staging | --dry-run | - # ------------------------------------------------------------ - # | --server acme-v02 | Use staging | Use staging | - # | --server acme-staging-v02 | Use staging | Use staging | - # | --server | Conflict error | Use | - - default_servers = (flag_default("server"), constants.STAGING_URI) - - if parsed_args.staging and parsed_args.server not in default_servers: - raise errors.Error("--server value conflicts with --staging") - - if parsed_args.server in default_servers: - parsed_args.server = constants.STAGING_URI - - if parsed_args.dry_run: - if self.verb not in ["certonly", "renew"]: - raise errors.Error("--dry-run currently only works with the " - "'certonly' or 'renew' subcommands (%r)" % self.verb) - parsed_args.break_my_certs = parsed_args.staging = True - if glob.glob(os.path.join(parsed_args.config_dir, constants.ACCOUNTS_DIR, "*")): - # The user has a prod account, but might not have a staging - # one; we don't want to start trying to perform interactive registration - parsed_args.tos = True - parsed_args.register_unsafely_without_email = True - - def handle_csr(self, parsed_args): - """Process a --csr flag.""" - if parsed_args.verb != "certonly": - raise errors.Error("Currently, a CSR file may only be specified " - "when obtaining a new or replacement " - "via the certonly command. Please try the " - "certonly command instead.") - if parsed_args.allow_subset_of_names: - raise errors.Error("--allow-subset-of-names cannot be used with --csr") - - csrfile, contents = parsed_args.csr[0:2] - typ, csr, domains = crypto_util.import_csr_file(csrfile, contents) - - # This is not necessary for webroot to work, however, - # obtain_certificate_from_csr requires parsed_args.domains to be set - for domain in domains: - add_domains(parsed_args, domain) - - if not domains: - # TODO: add CN to domains instead: - raise errors.Error( - "Unfortunately, your CSR %s needs to have a SubjectAltName for every domain" - % parsed_args.csr[0]) - - parsed_args.actual_csr = (csr, typ) - - csr_domains = {d.lower() for d in domains} - config_domains = set(parsed_args.domains) - if csr_domains != config_domains: - raise errors.ConfigurationError( - "Inconsistent domain requests:\nFrom the CSR: {0}\nFrom command line/config: {1}" - .format(", ".join(csr_domains), ", ".join(config_domains))) - - - def determine_verb(self): - """Determines the verb/subcommand provided by the user. - - This function works around some of the limitations of argparse. - - """ - if "-h" in self.args or "--help" in self.args: - # all verbs double as help arguments; don't get them confused - self.verb = "help" - return - - for i, token in enumerate(self.args): - if token in self.VERBS: - verb = token - if verb == "auth": - verb = "certonly" - if verb == "everything": - verb = "run" - self.verb = verb - self.args.pop(i) - return - - self.verb = "run" - - def prescan_for_flag(self, flag, possible_arguments): - """Checks cli input for flags. - - Check for a flag, which accepts a fixed set of possible arguments, in - the command line; we will use this information to configure argparse's - help correctly. Return the flag's argument, if it has one that matches - the sequence @possible_arguments; otherwise return whether the flag is - present. - - """ - if flag not in self.args: - return False - pos = self.args.index(flag) - try: - nxt = self.args[pos + 1] - if nxt in possible_arguments: - return nxt - except IndexError: - pass - return True - - def add(self, topics, *args, **kwargs): - """Add a new command line argument. - - :param topics: str or [str] help topic(s) this should be listed under, - or None for options that don't fit under a specific - topic which will only be shown in "--help all" output. - The first entry determines where the flag lives in the - "--help all" output (None -> "optional arguments"). - :param list *args: the names of this argument flag - :param dict **kwargs: various argparse settings for this argument - - """ - - if isinstance(topics, list): - # if this flag can be listed in multiple sections, try to pick the one - # that the user has asked for help about - topic = self.help_arg if self.help_arg in topics else topics[0] - else: - topic = topics # there's only one - - if self.detect_defaults: - kwargs = self.modify_kwargs_for_default_detection(**kwargs) - - if self.visible_topics[topic]: - if topic in self.groups: - group = self.groups[topic] - group.add_argument(*args, **kwargs) - else: - self.parser.add_argument(*args, **kwargs) - else: - kwargs["help"] = argparse.SUPPRESS - self.parser.add_argument(*args, **kwargs) - - def modify_kwargs_for_default_detection(self, **kwargs): - """Modify an arg so we can check if it was set by the user. - - Changes the parameters given to argparse when adding an argument - so we can properly detect if the value was set by the user. - - :param dict kwargs: various argparse settings for this argument - - :returns: a modified versions of kwargs - :rtype: dict - - """ - action = kwargs.get("action", None) - if action not in EXIT_ACTIONS: - kwargs["action"] = ("store_true" if action in ZERO_ARG_ACTIONS else - "store") - kwargs["default"] = _Default() - for param in ARGPARSE_PARAMS_TO_REMOVE: - kwargs.pop(param, None) - - return kwargs - - def add_deprecated_argument(self, argument_name, num_args): - """Adds a deprecated argument with the name argument_name. - - Deprecated arguments are not shown in the help. If they are used - on the command line, a warning is shown stating that the - argument is deprecated and no other action is taken. - - :param str argument_name: Name of deprecated argument. - :param int nargs: Number of arguments the option takes. - - """ - util.add_deprecated_argument( - self.parser.add_argument, argument_name, num_args) - - def add_group(self, topic, verbs=(), **kwargs): - """Create a new argument group. - - This method must be called once for every topic, however, calls - to this function are left next to the argument definitions for - clarity. - - :param str topic: Name of the new argument group. - :param str verbs: List of subcommands that should be documented as part of - this help group / topic - - :returns: The new argument group. - :rtype: `HelpfulArgumentGroup` - - """ - if self.visible_topics[topic]: - self.groups[topic] = self.parser.add_argument_group(topic, **kwargs) - if self.help_arg: - for v in verbs: - self.groups[topic].add_argument(v, help=VERB_HELP_MAP[v]["short"]) - return HelpfulArgumentGroup(self, topic) - - def add_plugin_args(self, plugins): - """ - - Let each of the plugins add its own command line arguments, which - may or may not be displayed as help topics. - - """ - for name, plugin_ep in six.iteritems(plugins): - parser_or_group = self.add_group(name, - description=plugin_ep.long_description) - plugin_ep.plugin_cls.inject_parser_options(parser_or_group, name) - - def determine_help_topics(self, chosen_topic): - """ - - The user may have requested help on a topic, return a dict of which - topics to display. @chosen_topic has prescan_for_flag's return type - - :returns: dict - - """ - # topics maps each topic to whether it should be documented by - # argparse on the command line - if chosen_topic == "auth": - chosen_topic = "certonly" - if chosen_topic == "everything": - chosen_topic = "run" - if chosen_topic == "all": - # Addition of condition closes #6209 (removal of duplicate route53 option). - return {t: t != 'certbot-route53:auth' for t in self.help_topics} - elif not chosen_topic: - return {t: False for t in self.help_topics} - return {t: t == chosen_topic for t in self.help_topics} - - -def _add_all_groups(helpful): - helpful.add_group("automation", description="Flags for automating execution & other tweaks") - helpful.add_group("security", description="Security parameters & server settings") - helpful.add_group("testing", - description="The following flags are meant for testing and integration purposes only.") - helpful.add_group("paths", description="Flags for changing execution paths & servers") - helpful.add_group("manage", - description="Various subcommands and flags are available for managing your certificates:", - verbs=["certificates", "delete", "renew", "revoke", "update_symlinks"]) - - # VERBS - for verb, docs in VERB_HELP: - name = docs.get("realname", verb) - helpful.add_group(name, description=docs["opts"]) - - -def prepare_and_parse_args(plugins, args, detect_defaults=False): - """Returns parsed command line arguments. - - :param .PluginsRegistry plugins: available plugins - :param list args: command line arguments with the program name removed - - :returns: parsed command line arguments - :rtype: argparse.Namespace - - """ - - - helpful = HelpfulArgumentParser(args, plugins, detect_defaults) - _add_all_groups(helpful) - - # --help is automatically provided by argparse - helpful.add( - None, "-v", "--verbose", dest="verbose_count", action="count", - default=flag_default("verbose_count"), help="This flag can be used " - "multiple times to incrementally increase the verbosity of output, " - "e.g. -vvv.") - helpful.add( - None, "-t", "--text", dest="text_mode", action="store_true", - default=flag_default("text_mode"), help=argparse.SUPPRESS) - helpful.add( - None, "--max-log-backups", type=nonnegative_int, - default=flag_default("max_log_backups"), - help="Specifies the maximum number of backup logs that should " - "be kept by Certbot's built in log rotation. Setting this " - "flag to 0 disables log rotation entirely, causing " - "Certbot to always append to the same log file.") - helpful.add( - [None, "automation", "run", "certonly", "enhance"], - "-n", "--non-interactive", "--noninteractive", - dest="noninteractive_mode", action="store_true", - default=flag_default("noninteractive_mode"), - help="Run without ever asking for user input. This may require " - "additional command line flags; the client will try to explain " - "which ones are required if it finds one missing") - helpful.add( - [None, "register", "run", "certonly", "enhance"], - constants.FORCE_INTERACTIVE_FLAG, action="store_true", - default=flag_default("force_interactive"), - help="Force Certbot to be interactive even if it detects it's not " - "being run in a terminal. This flag cannot be used with the " - "renew subcommand.") - helpful.add( - [None, "run", "certonly", "certificates", "enhance"], - "-d", "--domains", "--domain", dest="domains", - metavar="DOMAIN", action=_DomainsAction, - default=flag_default("domains"), - help="Domain names to apply. For multiple domains you can use " - "multiple -d flags or enter a comma separated list of domains " - "as a parameter. The first domain provided will be the " - "subject CN of the certificate, and all domains will be " - "Subject Alternative Names on the certificate. " - "The first domain will also be used in " - "some software user interfaces and as the file paths for the " - "certificate and related material unless otherwise " - "specified or you already have a certificate with the same " - "name. In the case of a name collision it will append a number " - "like 0001 to the file path name. (default: Ask)") - helpful.add( - [None, "run", "certonly", "register"], - "--eab-kid", dest="eab_kid", - metavar="EAB_KID", - help="Key Identifier for External Account Binding" - ) - helpful.add( - [None, "run", "certonly", "register"], - "--eab-hmac-key", dest="eab_hmac_key", - metavar="EAB_HMAC_KEY", - help="HMAC key for External Account Binding" - ) - helpful.add( - [None, "run", "certonly", "manage", "delete", "certificates", - "renew", "enhance"], "--cert-name", dest="certname", - metavar="CERTNAME", default=flag_default("certname"), - help="Certificate name to apply. This name is used by Certbot for housekeeping " - "and in file paths; it doesn't affect the content of the certificate itself. " - "To see certificate names, run 'certbot certificates'. " - "When creating a new certificate, specifies the new certificate's name. " - "(default: the first provided domain or the name of an existing " - "certificate on your system for the same domains)") - helpful.add( - [None, "testing", "renew", "certonly"], - "--dry-run", action="store_true", dest="dry_run", - default=flag_default("dry_run"), - help="Perform a test run of the client, obtaining test (invalid) certificates" - " but not saving them to disk. This can currently only be used" - " with the 'certonly' and 'renew' subcommands. \nNote: Although --dry-run" - " tries to avoid making any persistent changes on a system, it " - " is not completely side-effect free: if used with webserver authenticator plugins" - " like apache and nginx, it makes and then reverts temporary config changes" - " in order to obtain test certificates, and reloads webservers to deploy and then" - " roll back those changes. It also calls --pre-hook and --post-hook commands" - " if they are defined because they may be necessary to accurately simulate" - " renewal. --deploy-hook commands are not called.") - helpful.add( - ["register", "automation"], "--register-unsafely-without-email", action="store_true", - default=flag_default("register_unsafely_without_email"), - help="Specifying this flag enables registering an account with no " - "email address. This is strongly discouraged, because in the " - "event of key loss or account compromise you will irrevocably " - "lose access to your account. You will also be unable to receive " - "notice about impending expiration or revocation of your " - "certificates. Updates to the Subscriber Agreement will still " - "affect you, and will be effective 14 days after posting an " - "update to the web site.") - helpful.add( - ["register", "update_account", "unregister", "automation"], "-m", "--email", - default=flag_default("email"), - help=config_help("email")) - helpful.add(["register", "update_account", "automation"], "--eff-email", action="store_true", - default=flag_default("eff_email"), dest="eff_email", - help="Share your e-mail address with EFF") - helpful.add(["register", "update_account", "automation"], "--no-eff-email", - action="store_false", default=flag_default("eff_email"), dest="eff_email", - help="Don't share your e-mail address with EFF") - helpful.add( - ["automation", "certonly", "run"], - "--keep-until-expiring", "--keep", "--reinstall", - dest="reinstall", action="store_true", default=flag_default("reinstall"), - help="If the requested certificate matches an existing certificate, always keep the " - "existing one until it is due for renewal (for the " - "'run' subcommand this means reinstall the existing certificate). (default: Ask)") - helpful.add( - "automation", "--expand", action="store_true", default=flag_default("expand"), - help="If an existing certificate is a strict subset of the requested names, " - "always expand and replace it with the additional names. (default: Ask)") - helpful.add( - "automation", "--version", action="version", - version="%(prog)s {0}".format(certbot.__version__), - help="show program's version number and exit") - helpful.add( - ["automation", "renew"], - "--force-renewal", "--renew-by-default", dest="renew_by_default", - action="store_true", default=flag_default("renew_by_default"), - help="If a certificate " - "already exists for the requested domains, renew it now, " - "regardless of whether it is near expiry. (Often " - "--keep-until-expiring is more appropriate). Also implies " - "--expand.") - helpful.add( - "automation", "--renew-with-new-domains", dest="renew_with_new_domains", - action="store_true", default=flag_default("renew_with_new_domains"), - help="If a " - "certificate already exists for the requested certificate name " - "but does not match the requested domains, renew it now, " - "regardless of whether it is near expiry.") - helpful.add( - "automation", "--reuse-key", dest="reuse_key", - action="store_true", default=flag_default("reuse_key"), - help="When renewing, use the same private key as the existing " - "certificate.") - - helpful.add( - ["automation", "renew", "certonly"], - "--allow-subset-of-names", action="store_true", - default=flag_default("allow_subset_of_names"), - help="When performing domain validation, do not consider it a failure " - "if authorizations can not be obtained for a strict subset of " - "the requested domains. This may be useful for allowing renewals for " - "multiple domains to succeed even if some domains no longer point " - "at this system. This option cannot be used with --csr.") - helpful.add( - "automation", "--agree-tos", dest="tos", action="store_true", - default=flag_default("tos"), - help="Agree to the ACME Subscriber Agreement (default: Ask)") - helpful.add( - ["unregister", "automation"], "--account", metavar="ACCOUNT_ID", - default=flag_default("account"), - help="Account ID to use") - helpful.add( - "automation", "--duplicate", dest="duplicate", action="store_true", - default=flag_default("duplicate"), - help="Allow making a certificate lineage that duplicates an existing one " - "(both can be renewed in parallel)") - helpful.add( - "automation", "--os-packages-only", action="store_true", - default=flag_default("os_packages_only"), - help="(certbot-auto only) install OS package dependencies and then stop") - helpful.add( - "automation", "--no-self-upgrade", action="store_true", - default=flag_default("no_self_upgrade"), - help="(certbot-auto only) prevent the certbot-auto script from" - " upgrading itself to newer released versions (default: Upgrade" - " automatically)") - helpful.add( - "automation", "--no-bootstrap", action="store_true", - default=flag_default("no_bootstrap"), - help="(certbot-auto only) prevent the certbot-auto script from" - " installing OS-level dependencies (default: Prompt to install " - " OS-wide dependencies, but exit if the user says 'No')") - helpful.add( - "automation", "--no-permissions-check", action="store_true", - default=flag_default("no_permissions_check"), - help="(certbot-auto only) skip the check on the file system" - " permissions of the certbot-auto script") - helpful.add( - ["automation", "renew", "certonly", "run"], - "-q", "--quiet", dest="quiet", action="store_true", - default=flag_default("quiet"), - help="Silence all output except errors. Useful for automation via cron." - " Implies --non-interactive.") - # overwrites server, handled in HelpfulArgumentParser.parse_args() - helpful.add(["testing", "revoke", "run"], "--test-cert", "--staging", - dest="staging", action="store_true", default=flag_default("staging"), - help="Use the staging server to obtain or revoke test (invalid) certificates; equivalent" - " to --server " + constants.STAGING_URI) - helpful.add( - "testing", "--debug", action="store_true", default=flag_default("debug"), - help="Show tracebacks in case of errors, and allow certbot-auto " - "execution on experimental platforms") - helpful.add( - [None, "certonly", "run"], "--debug-challenges", action="store_true", - default=flag_default("debug_challenges"), - help="After setting up challenges, wait for user input before " - "submitting to CA") - helpful.add( - "testing", "--no-verify-ssl", action="store_true", - help=config_help("no_verify_ssl"), - default=flag_default("no_verify_ssl")) - helpful.add( - ["testing", "standalone", "manual"], "--http-01-port", type=int, - dest="http01_port", - default=flag_default("http01_port"), help=config_help("http01_port")) - helpful.add( - ["testing", "standalone"], "--http-01-address", - dest="http01_address", - default=flag_default("http01_address"), help=config_help("http01_address")) - helpful.add( - ["testing", "nginx"], "--https-port", type=int, - default=flag_default("https_port"), - help=config_help("https_port")) - helpful.add( - "testing", "--break-my-certs", action="store_true", - default=flag_default("break_my_certs"), - help="Be willing to replace or renew valid certificates with invalid " - "(testing/staging) certificates") - helpful.add( - "security", "--rsa-key-size", type=int, metavar="N", - default=flag_default("rsa_key_size"), help=config_help("rsa_key_size")) - helpful.add( - "security", "--must-staple", action="store_true", - dest="must_staple", default=flag_default("must_staple"), - help=config_help("must_staple")) - helpful.add( - ["security", "enhance"], - "--redirect", action="store_true", dest="redirect", - default=flag_default("redirect"), - help="Automatically redirect all HTTP traffic to HTTPS for the newly " - "authenticated vhost. (default: Ask)") - helpful.add( - "security", "--no-redirect", action="store_false", dest="redirect", - default=flag_default("redirect"), - help="Do not automatically redirect all HTTP traffic to HTTPS for the newly " - "authenticated vhost. (default: Ask)") - helpful.add( - ["security", "enhance"], - "--hsts", action="store_true", dest="hsts", default=flag_default("hsts"), - help="Add the Strict-Transport-Security header to every HTTP response." - " Forcing browser to always use SSL for the domain." - " Defends against SSL Stripping.") - helpful.add( - "security", "--no-hsts", action="store_false", dest="hsts", - default=flag_default("hsts"), help=argparse.SUPPRESS) - helpful.add( - ["security", "enhance"], - "--uir", action="store_true", dest="uir", default=flag_default("uir"), - help='Add the "Content-Security-Policy: upgrade-insecure-requests"' - ' header to every HTTP response. Forcing the browser to use' - ' https:// for every http:// resource.') - helpful.add( - "security", "--no-uir", action="store_false", dest="uir", default=flag_default("uir"), - help=argparse.SUPPRESS) - helpful.add( - "security", "--staple-ocsp", action="store_true", dest="staple", - default=flag_default("staple"), - help="Enables OCSP Stapling. A valid OCSP response is stapled to" - " the certificate that the server offers during TLS.") - helpful.add( - "security", "--no-staple-ocsp", action="store_false", dest="staple", - default=flag_default("staple"), help=argparse.SUPPRESS) - helpful.add( - "security", "--strict-permissions", action="store_true", - default=flag_default("strict_permissions"), - help="Require that all configuration files are owned by the current " - "user; only needed if your config is somewhere unsafe like /tmp/") - helpful.add( - ["manual", "standalone", "certonly", "renew"], - "--preferred-challenges", dest="pref_challs", - action=_PrefChallAction, default=flag_default("pref_challs"), - help='A sorted, comma delimited list of the preferred challenge to ' - 'use during authorization with the most preferred challenge ' - 'listed first (Eg, "dns" or "http,dns"). ' - 'Not all plugins support all challenges. See ' - 'https://certbot.eff.org/docs/using.html#plugins for details. ' - 'ACME Challenges are versioned, but if you pick "http" rather ' - 'than "http-01", Certbot will select the latest version ' - 'automatically.') - helpful.add( - "renew", "--pre-hook", - help="Command to be run in a shell before obtaining any certificates." - " Intended primarily for renewal, where it can be used to temporarily" - " shut down a webserver that might conflict with the standalone" - " plugin. This will only be called if a certificate is actually to be" - " obtained/renewed. When renewing several certificates that have" - " identical pre-hooks, only the first will be executed.") - helpful.add( - "renew", "--post-hook", - help="Command to be run in a shell after attempting to obtain/renew" - " certificates. Can be used to deploy renewed certificates, or to" - " restart any servers that were stopped by --pre-hook. This is only" - " run if an attempt was made to obtain/renew a certificate. If" - " multiple renewed certificates have identical post-hooks, only" - " one will be run.") - helpful.add("renew", "--renew-hook", - action=_RenewHookAction, help=argparse.SUPPRESS) - helpful.add( - "renew", "--no-random-sleep-on-renew", action="store_false", - default=flag_default("random_sleep_on_renew"), dest="random_sleep_on_renew", - help=argparse.SUPPRESS) - helpful.add( - "renew", "--deploy-hook", action=_DeployHookAction, - help='Command to be run in a shell once for each successfully' - ' issued certificate. For this command, the shell variable' - ' $RENEWED_LINEAGE will point to the config live subdirectory' - ' (for example, "/etc/letsencrypt/live/example.com") containing' - ' the new certificates and keys; the shell variable' - ' $RENEWED_DOMAINS will contain a space-delimited list of' - ' renewed certificate domains (for example, "example.com' - ' www.example.com"') - helpful.add( - "renew", "--disable-hook-validation", - action="store_false", dest="validate_hooks", - default=flag_default("validate_hooks"), - help="Ordinarily the commands specified for" - " --pre-hook/--post-hook/--deploy-hook will be checked for" - " validity, to see if the programs being run are in the $PATH," - " so that mistakes can be caught early, even when the hooks" - " aren't being run just yet. The validation is rather" - " simplistic and fails if you use more advanced shell" - " constructs, so you can use this switch to disable it." - " (default: False)") - helpful.add( - "renew", "--no-directory-hooks", action="store_false", - default=flag_default("directory_hooks"), dest="directory_hooks", - help="Disable running executables found in Certbot's hook directories" - " during renewal. (default: False)") - helpful.add( - "renew", "--disable-renew-updates", action="store_true", - default=flag_default("disable_renew_updates"), dest="disable_renew_updates", - help="Disable automatic updates to your server configuration that" - " would otherwise be done by the selected installer plugin, and triggered" - " when the user executes \"certbot renew\", regardless of if the certificate" - " is renewed. This setting does not apply to important TLS configuration" - " updates.") - helpful.add( - "renew", "--no-autorenew", action="store_false", - default=flag_default("autorenew"), dest="autorenew", - help="Disable auto renewal of certificates.") - - # Populate the command line parameters for new style enhancements - enhancements.populate_cli(helpful.add) - - _create_subparsers(helpful) - _paths_parser(helpful) - # _plugins_parsing should be the last thing to act upon the main - # parser (--help should display plugin-specific options last) - _plugins_parsing(helpful, plugins) - - if not detect_defaults: - global helpful_parser # pylint: disable=global-statement - helpful_parser = helpful - return helpful.parse_args() - - -def _create_subparsers(helpful): - from certbot._internal.client import sample_user_agent # avoid import loops - helpful.add( - None, "--user-agent", default=flag_default("user_agent"), - help='Set a custom user agent string for the client. User agent strings allow ' - 'the CA to collect high level statistics about success rates by OS, ' - 'plugin and use case, and to know when to deprecate support for past Python ' - "versions and flags. If you wish to hide this information from the Let's " - 'Encrypt server, set this to "". ' - '(default: {0}). The flags encoded in the user agent are: ' - '--duplicate, --force-renew, --allow-subset-of-names, -n, and ' - 'whether any hooks are set.'.format(sample_user_agent())) - helpful.add( - None, "--user-agent-comment", default=flag_default("user_agent_comment"), - type=_user_agent_comment_type, - help="Add a comment to the default user agent string. May be used when repackaging Certbot " - "or calling it from another tool to allow additional statistical data to be collected." - " Ignored if --user-agent is set. (Example: Foo-Wrapper/1.0)") - helpful.add("certonly", - "--csr", default=flag_default("csr"), type=read_file, - help="Path to a Certificate Signing Request (CSR) in DER or PEM format." - " Currently --csr only works with the 'certonly' subcommand.") - helpful.add("revoke", - "--reason", dest="reason", - choices=CaseInsensitiveList(sorted(constants.REVOCATION_REASONS, - key=constants.REVOCATION_REASONS.get)), - action=_EncodeReasonAction, default=flag_default("reason"), - help="Specify reason for revoking certificate. (default: unspecified)") - helpful.add("revoke", - "--delete-after-revoke", action="store_true", - default=flag_default("delete_after_revoke"), - help="Delete certificates after revoking them, along with all previous and later " - "versions of those certificates.") - helpful.add("revoke", - "--no-delete-after-revoke", action="store_false", - dest="delete_after_revoke", - default=flag_default("delete_after_revoke"), - help="Do not delete certificates after revoking them. This " - "option should be used with caution because the 'renew' " - "subcommand will attempt to renew undeleted revoked " - "certificates.") - helpful.add("rollback", - "--checkpoints", type=int, metavar="N", - default=flag_default("rollback_checkpoints"), - help="Revert configuration N number of checkpoints.") - helpful.add("plugins", - "--init", action="store_true", default=flag_default("init"), - help="Initialize plugins.") - helpful.add("plugins", - "--prepare", action="store_true", default=flag_default("prepare"), - help="Initialize and prepare plugins.") - helpful.add("plugins", - "--authenticators", action="append_const", dest="ifaces", - default=flag_default("ifaces"), - const=interfaces.IAuthenticator, help="Limit to authenticator plugins only.") - helpful.add("plugins", - "--installers", action="append_const", dest="ifaces", - default=flag_default("ifaces"), - const=interfaces.IInstaller, help="Limit to installer plugins only.") - - -class CaseInsensitiveList(list): - """A list that will ignore case when searching. - - This class is passed to the `choices` argument of `argparse.add_arguments` - through the `helpful` wrapper. It is necessary due to special handling of - command line arguments by `set_by_cli` in which the `type_func` is not applied.""" - def __contains__(self, element): - return super(CaseInsensitiveList, self).__contains__(element.lower()) - - -def _paths_parser(helpful): - add = helpful.add - verb = helpful.verb - if verb == "help": - verb = helpful.help_arg - - cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked." - sections = ["paths", "install", "revoke", "certonly", "manage"] - if verb == "certonly": - add(sections, "--cert-path", type=os.path.abspath, - default=flag_default("auth_cert_path"), help=cph) - elif verb == "revoke": - add(sections, "--cert-path", type=read_file, required=False, help=cph) - else: - add(sections, "--cert-path", type=os.path.abspath, help=cph) - - section = "paths" - if verb in ("install", "revoke"): - section = verb - # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", - type=((verb == "revoke" and read_file) or os.path.abspath), - help="Path to private key for certificate installation " - "or revocation (if account key is missing)") - - default_cp = None - if verb == "certonly": - default_cp = flag_default("auth_chain_path") - add(["paths", "install"], "--fullchain-path", default=default_cp, type=os.path.abspath, - help="Accompanying path to a full certificate chain (certificate plus chain).") - add("paths", "--chain-path", default=default_cp, type=os.path.abspath, - help="Accompanying path to a certificate chain.") - add("paths", "--config-dir", default=flag_default("config_dir"), - help=config_help("config_dir")) - add("paths", "--work-dir", default=flag_default("work_dir"), - help=config_help("work_dir")) - add("paths", "--logs-dir", default=flag_default("logs_dir"), - help="Logs directory.") - add("paths", "--server", default=flag_default("server"), - help=config_help("server")) - - -def _plugins_parsing(helpful, plugins): - # It's nuts, but there are two "plugins" topics. Somehow this works - helpful.add_group( - "plugins", description="Plugin Selection: Certbot client supports an " - "extensible plugins architecture. See '%(prog)s plugins' for a " - "list of all installed plugins and their names. You can force " - "a particular plugin by setting options provided below. Running " - "--help will list flags specific to that plugin.") - - helpful.add("plugins", "--configurator", default=flag_default("configurator"), - help="Name of the plugin that is both an authenticator and an installer." - " Should not be used together with --authenticator or --installer. " - "(default: Ask)") - helpful.add("plugins", "-a", "--authenticator", default=flag_default("authenticator"), - help="Authenticator plugin name.") - helpful.add("plugins", "-i", "--installer", default=flag_default("installer"), - help="Installer plugin name (also used to find domains).") - helpful.add(["plugins", "certonly", "run", "install"], - "--apache", action="store_true", default=flag_default("apache"), - help="Obtain and install certificates using Apache") - helpful.add(["plugins", "certonly", "run", "install"], - "--nginx", action="store_true", default=flag_default("nginx"), - help="Obtain and install certificates using Nginx") - helpful.add(["plugins", "certonly"], "--standalone", action="store_true", - default=flag_default("standalone"), - help='Obtain certificates using a "standalone" webserver.') - helpful.add(["plugins", "certonly"], "--manual", action="store_true", - default=flag_default("manual"), - help="Provide laborious manual instructions for obtaining a certificate") - helpful.add(["plugins", "certonly"], "--webroot", action="store_true", - default=flag_default("webroot"), - help="Obtain certificates by placing files in a webroot directory.") - helpful.add(["plugins", "certonly"], "--dns-cloudflare", action="store_true", - default=flag_default("dns_cloudflare"), - help=("Obtain certificates using a DNS TXT record (if you are " - "using Cloudflare for DNS).")) - helpful.add(["plugins", "certonly"], "--dns-cloudxns", action="store_true", - default=flag_default("dns_cloudxns"), - help=("Obtain certificates using a DNS TXT record (if you are " - "using CloudXNS for DNS).")) - helpful.add(["plugins", "certonly"], "--dns-digitalocean", action="store_true", - default=flag_default("dns_digitalocean"), - help=("Obtain certificates using a DNS TXT record (if you are " - "using DigitalOcean for DNS).")) - helpful.add(["plugins", "certonly"], "--dns-dnsimple", action="store_true", - default=flag_default("dns_dnsimple"), - help=("Obtain certificates using a DNS TXT record (if you are " - "using DNSimple for DNS).")) - helpful.add(["plugins", "certonly"], "--dns-dnsmadeeasy", action="store_true", - default=flag_default("dns_dnsmadeeasy"), - help=("Obtain certificates using a DNS TXT record (if you are " - "using DNS Made Easy for DNS).")) - helpful.add(["plugins", "certonly"], "--dns-gehirn", action="store_true", - default=flag_default("dns_gehirn"), - help=("Obtain certificates using a DNS TXT record " - "(if you are using Gehirn Infrastructure Service for DNS).")) - helpful.add(["plugins", "certonly"], "--dns-google", action="store_true", - default=flag_default("dns_google"), - help=("Obtain certificates using a DNS TXT record (if you are " - "using Google Cloud DNS).")) - helpful.add(["plugins", "certonly"], "--dns-linode", action="store_true", - default=flag_default("dns_linode"), - help=("Obtain certificates using a DNS TXT record (if you are " - "using Linode for DNS).")) - helpful.add(["plugins", "certonly"], "--dns-luadns", action="store_true", - default=flag_default("dns_luadns"), - help=("Obtain certificates using a DNS TXT record (if you are " - "using LuaDNS for DNS).")) - helpful.add(["plugins", "certonly"], "--dns-nsone", action="store_true", - default=flag_default("dns_nsone"), - help=("Obtain certificates using a DNS TXT record (if you are " - "using NS1 for DNS).")) - helpful.add(["plugins", "certonly"], "--dns-ovh", action="store_true", - default=flag_default("dns_ovh"), - help=("Obtain certificates using a DNS TXT record (if you are " - "using OVH for DNS).")) - helpful.add(["plugins", "certonly"], "--dns-rfc2136", action="store_true", - default=flag_default("dns_rfc2136"), - help="Obtain certificates using a DNS TXT record (if you are using BIND for DNS).") - helpful.add(["plugins", "certonly"], "--dns-route53", action="store_true", - default=flag_default("dns_route53"), - help=("Obtain certificates using a DNS TXT record (if you are using Route53 for " - "DNS).")) - helpful.add(["plugins", "certonly"], "--dns-sakuracloud", action="store_true", - default=flag_default("dns_sakuracloud"), - help=("Obtain certificates using a DNS TXT record " - "(if you are using Sakura Cloud for DNS).")) - - # things should not be reorder past/pre this comment: - # plugins_group should be displayed in --help before plugin - # specific groups (so that plugins_group.description makes sense) - - helpful.add_plugin_args(plugins) - - -class _EncodeReasonAction(argparse.Action): - """Action class for parsing revocation reason.""" - - def __call__(self, parser, namespace, reason, option_string=None): - """Encodes the reason for certificate revocation.""" - code = constants.REVOCATION_REASONS[reason.lower()] - setattr(namespace, self.dest, code) - - -class _DomainsAction(argparse.Action): - """Action class for parsing domains.""" - - def __call__(self, parser, namespace, domain, option_string=None): - """Just wrap add_domains in argparseese.""" - add_domains(namespace, domain) - -def add_domains(args_or_config, domains): - """Registers new domains to be used during the current client run. - - Domains are not added to the list of requested domains if they have - already been registered. - - :param args_or_config: parsed command line arguments - :type args_or_config: argparse.Namespace or - configuration.NamespaceConfig - :param str domain: one or more comma separated domains - - :returns: domains after they have been normalized and validated - :rtype: `list` of `str` - - """ - validated_domains = [] - for domain in domains.split(","): - domain = util.enforce_domain_sanity(domain.strip()) - validated_domains.append(domain) - if domain not in args_or_config.domains: - args_or_config.domains.append(domain) - - return validated_domains - -class _PrefChallAction(argparse.Action): - """Action class for parsing preferred challenges.""" - - def __call__(self, parser, namespace, pref_challs, option_string=None): - try: - challs = parse_preferred_challenges(pref_challs.split(",")) - except errors.Error as error: - raise argparse.ArgumentError(self, str(error)) - namespace.pref_challs.extend(challs) - - -def parse_preferred_challenges(pref_challs): - """Translate and validate preferred challenges. - - :param pref_challs: list of preferred challenge types - :type pref_challs: `list` of `str` - - :returns: validated list of preferred challenge types - :rtype: `list` of `str` - - :raises errors.Error: if pref_challs is invalid - - """ - aliases = {"dns": "dns-01", "http": "http-01"} - challs = [c.strip() for c in pref_challs] - challs = [aliases.get(c, c) for c in challs] - - unrecognized = ", ".join(name for name in challs - if name not in challenges.Challenge.TYPES) - if unrecognized: - raise errors.Error( - "Unrecognized challenges: {0}".format(unrecognized)) - return challs - - -def _user_agent_comment_type(value): - if "(" in value or ")" in value: - raise argparse.ArgumentTypeError("may not contain parentheses") - return value - - -class _DeployHookAction(argparse.Action): - """Action class for parsing deploy hooks.""" - - def __call__(self, parser, namespace, values, option_string=None): - renew_hook_set = namespace.deploy_hook != namespace.renew_hook - if renew_hook_set and namespace.renew_hook != values: - raise argparse.ArgumentError( - self, "conflicts with --renew-hook value") - namespace.deploy_hook = namespace.renew_hook = values - - -class _RenewHookAction(argparse.Action): - """Action class for parsing renew hooks.""" - - def __call__(self, parser, namespace, values, option_string=None): - deploy_hook_set = namespace.deploy_hook is not None - if deploy_hook_set and namespace.deploy_hook != values: - raise argparse.ArgumentError( - self, "conflicts with --deploy-hook value") - namespace.renew_hook = values - - -def nonnegative_int(value): - """Converts value to an int and checks that it is not negative. - - This function should used as the type parameter for argparse - arguments. - - :param str value: value provided on the command line - - :returns: integer representation of value - :rtype: int - - :raises argparse.ArgumentTypeError: if value isn't a non-negative integer - - """ - try: - int_value = int(value) - except ValueError: - raise argparse.ArgumentTypeError("value must be an integer") - - if int_value < 0: - raise argparse.ArgumentTypeError("value must be non-negative") - return int_value diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py new file mode 100644 index 000000000..96dfb163e --- /dev/null +++ b/certbot/certbot/_internal/cli/__init__.py @@ -0,0 +1,526 @@ +"""Certbot command line argument & config processing.""" +# pylint: disable=too-many-lines +from __future__ import print_function +import logging +import logging.handlers +import argparse +import sys +import certbot._internal.plugins.selection as plugin_selection +from certbot._internal.plugins import disco as plugins_disco + +from acme.magic_typing import Optional + +# pylint: disable=ungrouped-imports +import certbot +from certbot._internal import constants + +import certbot.plugins.enhancements as enhancements + + +from certbot._internal.cli.cli_constants import ( + LEAUTO, + old_path_fragment, + new_path_prefix, + cli_command, + SHORT_USAGE, + COMMAND_OVERVIEW, + HELP_AND_VERSION_USAGE, + ARGPARSE_PARAMS_TO_REMOVE, + EXIT_ACTIONS, + ZERO_ARG_ACTIONS, + VAR_MODIFIERS +) + +from certbot._internal.cli.cli_utils import ( + _Default, + read_file, + flag_default, + config_help, + HelpfulArgumentGroup, + CustomHelpFormatter, + _DomainsAction, + add_domains, + CaseInsensitiveList, + _user_agent_comment_type, + _EncodeReasonAction, + parse_preferred_challenges, + _PrefChallAction, + _DeployHookAction, + _RenewHookAction, + nonnegative_int +) + +# These imports depend on cli_constants and cli_utils. +from certbot._internal.cli.report_config_interaction import report_config_interaction +from certbot._internal.cli.verb_help import VERB_HELP, VERB_HELP_MAP +from certbot._internal.cli.group_adder import _add_all_groups +from certbot._internal.cli.subparsers import _create_subparsers +from certbot._internal.cli.paths_parser import _paths_parser +from certbot._internal.cli.plugins_parsing import _plugins_parsing + +# These imports depend on some or all of the submodules for cli. +from certbot._internal.cli.helpful import HelpfulArgumentParser +# pylint: enable=ungrouped-imports + + +logger = logging.getLogger(__name__) + + +# Global, to save us from a lot of argument passing within the scope of this module +helpful_parser = None # type: Optional[HelpfulArgumentParser] + + +def prepare_and_parse_args(plugins, args, detect_defaults=False): + """Returns parsed command line arguments. + + :param .PluginsRegistry plugins: available plugins + :param list args: command line arguments with the program name removed + + :returns: parsed command line arguments + :rtype: argparse.Namespace + + """ + + helpful = HelpfulArgumentParser(args, plugins, detect_defaults) + _add_all_groups(helpful) + + # --help is automatically provided by argparse + helpful.add( + None, "-v", "--verbose", dest="verbose_count", action="count", + default=flag_default("verbose_count"), help="This flag can be used " + "multiple times to incrementally increase the verbosity of output, " + "e.g. -vvv.") + helpful.add( + None, "-t", "--text", dest="text_mode", action="store_true", + default=flag_default("text_mode"), help=argparse.SUPPRESS) + helpful.add( + None, "--max-log-backups", type=nonnegative_int, + default=flag_default("max_log_backups"), + help="Specifies the maximum number of backup logs that should " + "be kept by Certbot's built in log rotation. Setting this " + "flag to 0 disables log rotation entirely, causing " + "Certbot to always append to the same log file.") + helpful.add( + [None, "automation", "run", "certonly", "enhance"], + "-n", "--non-interactive", "--noninteractive", + dest="noninteractive_mode", action="store_true", + default=flag_default("noninteractive_mode"), + help="Run without ever asking for user input. This may require " + "additional command line flags; the client will try to explain " + "which ones are required if it finds one missing") + helpful.add( + [None, "register", "run", "certonly", "enhance"], + constants.FORCE_INTERACTIVE_FLAG, action="store_true", + default=flag_default("force_interactive"), + help="Force Certbot to be interactive even if it detects it's not " + "being run in a terminal. This flag cannot be used with the " + "renew subcommand.") + helpful.add( + [None, "run", "certonly", "certificates", "enhance"], + "-d", "--domains", "--domain", dest="domains", + metavar="DOMAIN", action=_DomainsAction, + default=flag_default("domains"), + help="Domain names to apply. For multiple domains you can use " + "multiple -d flags or enter a comma separated list of domains " + "as a parameter. The first domain provided will be the " + "subject CN of the certificate, and all domains will be " + "Subject Alternative Names on the certificate. " + "The first domain will also be used in " + "some software user interfaces and as the file paths for the " + "certificate and related material unless otherwise " + "specified or you already have a certificate with the same " + "name. In the case of a name collision it will append a number " + "like 0001 to the file path name. (default: Ask)") + helpful.add( + [None, "run", "certonly", "register"], + "--eab-kid", dest="eab_kid", + metavar="EAB_KID", + help="Key Identifier for External Account Binding" + ) + helpful.add( + [None, "run", "certonly", "register"], + "--eab-hmac-key", dest="eab_hmac_key", + metavar="EAB_HMAC_KEY", + help="HMAC key for External Account Binding" + ) + helpful.add( + [None, "run", "certonly", "manage", "delete", "certificates", + "renew", "enhance"], "--cert-name", dest="certname", + metavar="CERTNAME", default=flag_default("certname"), + help="Certificate name to apply. This name is used by Certbot for housekeeping " + "and in file paths; it doesn't affect the content of the certificate itself. " + "To see certificate names, run 'certbot certificates'. " + "When creating a new certificate, specifies the new certificate's name. " + "(default: the first provided domain or the name of an existing " + "certificate on your system for the same domains)") + helpful.add( + [None, "testing", "renew", "certonly"], + "--dry-run", action="store_true", dest="dry_run", + default=flag_default("dry_run"), + help="Perform a test run of the client, obtaining test (invalid) certificates" + " but not saving them to disk. This can currently only be used" + " with the 'certonly' and 'renew' subcommands. \nNote: Although --dry-run" + " tries to avoid making any persistent changes on a system, it " + " is not completely side-effect free: if used with webserver authenticator plugins" + " like apache and nginx, it makes and then reverts temporary config changes" + " in order to obtain test certificates, and reloads webservers to deploy and then" + " roll back those changes. It also calls --pre-hook and --post-hook commands" + " if they are defined because they may be necessary to accurately simulate" + " renewal. --deploy-hook commands are not called.") + helpful.add( + ["register", "automation"], "--register-unsafely-without-email", action="store_true", + default=flag_default("register_unsafely_without_email"), + help="Specifying this flag enables registering an account with no " + "email address. This is strongly discouraged, because in the " + "event of key loss or account compromise you will irrevocably " + "lose access to your account. You will also be unable to receive " + "notice about impending expiration or revocation of your " + "certificates. Updates to the Subscriber Agreement will still " + "affect you, and will be effective 14 days after posting an " + "update to the web site.") + helpful.add( + ["register", "update_account", "unregister", "automation"], "-m", "--email", + default=flag_default("email"), + help=config_help("email")) + helpful.add(["register", "update_account", "automation"], "--eff-email", action="store_true", + default=flag_default("eff_email"), dest="eff_email", + help="Share your e-mail address with EFF") + helpful.add(["register", "update_account", "automation"], "--no-eff-email", + action="store_false", default=flag_default("eff_email"), dest="eff_email", + help="Don't share your e-mail address with EFF") + helpful.add( + ["automation", "certonly", "run"], + "--keep-until-expiring", "--keep", "--reinstall", + dest="reinstall", action="store_true", default=flag_default("reinstall"), + help="If the requested certificate matches an existing certificate, always keep the " + "existing one until it is due for renewal (for the " + "'run' subcommand this means reinstall the existing certificate). (default: Ask)") + helpful.add( + "automation", "--expand", action="store_true", default=flag_default("expand"), + help="If an existing certificate is a strict subset of the requested names, " + "always expand and replace it with the additional names. (default: Ask)") + helpful.add( + "automation", "--version", action="version", + version="%(prog)s {0}".format(certbot.__version__), + help="show program's version number and exit") + helpful.add( + ["automation", "renew"], + "--force-renewal", "--renew-by-default", dest="renew_by_default", + action="store_true", default=flag_default("renew_by_default"), + help="If a certificate " + "already exists for the requested domains, renew it now, " + "regardless of whether it is near expiry. (Often " + "--keep-until-expiring is more appropriate). Also implies " + "--expand.") + helpful.add( + "automation", "--renew-with-new-domains", dest="renew_with_new_domains", + action="store_true", default=flag_default("renew_with_new_domains"), + help="If a " + "certificate already exists for the requested certificate name " + "but does not match the requested domains, renew it now, " + "regardless of whether it is near expiry.") + helpful.add( + "automation", "--reuse-key", dest="reuse_key", + action="store_true", default=flag_default("reuse_key"), + help="When renewing, use the same private key as the existing " + "certificate.") + + helpful.add( + ["automation", "renew", "certonly"], + "--allow-subset-of-names", action="store_true", + default=flag_default("allow_subset_of_names"), + help="When performing domain validation, do not consider it a failure " + "if authorizations can not be obtained for a strict subset of " + "the requested domains. This may be useful for allowing renewals for " + "multiple domains to succeed even if some domains no longer point " + "at this system. This option cannot be used with --csr.") + helpful.add( + "automation", "--agree-tos", dest="tos", action="store_true", + default=flag_default("tos"), + help="Agree to the ACME Subscriber Agreement (default: Ask)") + helpful.add( + ["unregister", "automation"], "--account", metavar="ACCOUNT_ID", + default=flag_default("account"), + help="Account ID to use") + helpful.add( + "automation", "--duplicate", dest="duplicate", action="store_true", + default=flag_default("duplicate"), + help="Allow making a certificate lineage that duplicates an existing one " + "(both can be renewed in parallel)") + helpful.add( + "automation", "--os-packages-only", action="store_true", + default=flag_default("os_packages_only"), + help="(certbot-auto only) install OS package dependencies and then stop") + helpful.add( + "automation", "--no-self-upgrade", action="store_true", + default=flag_default("no_self_upgrade"), + help="(certbot-auto only) prevent the certbot-auto script from" + " upgrading itself to newer released versions (default: Upgrade" + " automatically)") + helpful.add( + "automation", "--no-bootstrap", action="store_true", + default=flag_default("no_bootstrap"), + help="(certbot-auto only) prevent the certbot-auto script from" + " installing OS-level dependencies (default: Prompt to install " + " OS-wide dependencies, but exit if the user says 'No')") + helpful.add( + "automation", "--no-permissions-check", action="store_true", + default=flag_default("no_permissions_check"), + help="(certbot-auto only) skip the check on the file system" + " permissions of the certbot-auto script") + helpful.add( + ["automation", "renew", "certonly", "run"], + "-q", "--quiet", dest="quiet", action="store_true", + default=flag_default("quiet"), + help="Silence all output except errors. Useful for automation via cron." + " Implies --non-interactive.") + # overwrites server, handled in HelpfulArgumentParser.parse_args() + helpful.add(["testing", "revoke", "run"], "--test-cert", "--staging", + dest="staging", action="store_true", default=flag_default("staging"), + help="Use the staging server to obtain or revoke test (invalid) certificates; equivalent" + " to --server " + constants.STAGING_URI) + helpful.add( + "testing", "--debug", action="store_true", default=flag_default("debug"), + help="Show tracebacks in case of errors, and allow certbot-auto " + "execution on experimental platforms") + helpful.add( + [None, "certonly", "run"], "--debug-challenges", action="store_true", + default=flag_default("debug_challenges"), + help="After setting up challenges, wait for user input before " + "submitting to CA") + helpful.add( + "testing", "--no-verify-ssl", action="store_true", + help=config_help("no_verify_ssl"), + default=flag_default("no_verify_ssl")) + helpful.add( + ["testing", "standalone", "manual"], "--http-01-port", type=int, + dest="http01_port", + default=flag_default("http01_port"), help=config_help("http01_port")) + helpful.add( + ["testing", "standalone"], "--http-01-address", + dest="http01_address", + default=flag_default("http01_address"), help=config_help("http01_address")) + helpful.add( + ["testing", "nginx"], "--https-port", type=int, + default=flag_default("https_port"), + help=config_help("https_port")) + helpful.add( + "testing", "--break-my-certs", action="store_true", + default=flag_default("break_my_certs"), + help="Be willing to replace or renew valid certificates with invalid " + "(testing/staging) certificates") + helpful.add( + "security", "--rsa-key-size", type=int, metavar="N", + default=flag_default("rsa_key_size"), help=config_help("rsa_key_size")) + helpful.add( + "security", "--must-staple", action="store_true", + dest="must_staple", default=flag_default("must_staple"), + help=config_help("must_staple")) + helpful.add( + ["security", "enhance"], + "--redirect", action="store_true", dest="redirect", + default=flag_default("redirect"), + help="Automatically redirect all HTTP traffic to HTTPS for the newly " + "authenticated vhost. (default: Ask)") + helpful.add( + "security", "--no-redirect", action="store_false", dest="redirect", + default=flag_default("redirect"), + help="Do not automatically redirect all HTTP traffic to HTTPS for the newly " + "authenticated vhost. (default: Ask)") + helpful.add( + ["security", "enhance"], + "--hsts", action="store_true", dest="hsts", default=flag_default("hsts"), + help="Add the Strict-Transport-Security header to every HTTP response." + " Forcing browser to always use SSL for the domain." + " Defends against SSL Stripping.") + helpful.add( + "security", "--no-hsts", action="store_false", dest="hsts", + default=flag_default("hsts"), help=argparse.SUPPRESS) + helpful.add( + ["security", "enhance"], + "--uir", action="store_true", dest="uir", default=flag_default("uir"), + help='Add the "Content-Security-Policy: upgrade-insecure-requests"' + ' header to every HTTP response. Forcing the browser to use' + ' https:// for every http:// resource.') + helpful.add( + "security", "--no-uir", action="store_false", dest="uir", default=flag_default("uir"), + help=argparse.SUPPRESS) + helpful.add( + "security", "--staple-ocsp", action="store_true", dest="staple", + default=flag_default("staple"), + help="Enables OCSP Stapling. A valid OCSP response is stapled to" + " the certificate that the server offers during TLS.") + helpful.add( + "security", "--no-staple-ocsp", action="store_false", dest="staple", + default=flag_default("staple"), help=argparse.SUPPRESS) + helpful.add( + "security", "--strict-permissions", action="store_true", + default=flag_default("strict_permissions"), + help="Require that all configuration files are owned by the current " + "user; only needed if your config is somewhere unsafe like /tmp/") + helpful.add( + ["manual", "standalone", "certonly", "renew"], + "--preferred-challenges", dest="pref_challs", + action=_PrefChallAction, default=flag_default("pref_challs"), + help='A sorted, comma delimited list of the preferred challenge to ' + 'use during authorization with the most preferred challenge ' + 'listed first (Eg, "dns" or "http,dns"). ' + 'Not all plugins support all challenges. See ' + 'https://certbot.eff.org/docs/using.html#plugins for details. ' + 'ACME Challenges are versioned, but if you pick "http" rather ' + 'than "http-01", Certbot will select the latest version ' + 'automatically.') + helpful.add( + "renew", "--pre-hook", + help="Command to be run in a shell before obtaining any certificates." + " Intended primarily for renewal, where it can be used to temporarily" + " shut down a webserver that might conflict with the standalone" + " plugin. This will only be called if a certificate is actually to be" + " obtained/renewed. When renewing several certificates that have" + " identical pre-hooks, only the first will be executed.") + helpful.add( + "renew", "--post-hook", + help="Command to be run in a shell after attempting to obtain/renew" + " certificates. Can be used to deploy renewed certificates, or to" + " restart any servers that were stopped by --pre-hook. This is only" + " run if an attempt was made to obtain/renew a certificate. If" + " multiple renewed certificates have identical post-hooks, only" + " one will be run.") + helpful.add("renew", "--renew-hook", + action=_RenewHookAction, help=argparse.SUPPRESS) + helpful.add( + "renew", "--no-random-sleep-on-renew", action="store_false", + default=flag_default("random_sleep_on_renew"), dest="random_sleep_on_renew", + help=argparse.SUPPRESS) + helpful.add( + "renew", "--deploy-hook", action=_DeployHookAction, + help='Command to be run in a shell once for each successfully' + ' issued certificate. For this command, the shell variable' + ' $RENEWED_LINEAGE will point to the config live subdirectory' + ' (for example, "/etc/letsencrypt/live/example.com") containing' + ' the new certificates and keys; the shell variable' + ' $RENEWED_DOMAINS will contain a space-delimited list of' + ' renewed certificate domains (for example, "example.com' + ' www.example.com"') + helpful.add( + "renew", "--disable-hook-validation", + action="store_false", dest="validate_hooks", + default=flag_default("validate_hooks"), + help="Ordinarily the commands specified for" + " --pre-hook/--post-hook/--deploy-hook will be checked for" + " validity, to see if the programs being run are in the $PATH," + " so that mistakes can be caught early, even when the hooks" + " aren't being run just yet. The validation is rather" + " simplistic and fails if you use more advanced shell" + " constructs, so you can use this switch to disable it." + " (default: False)") + helpful.add( + "renew", "--no-directory-hooks", action="store_false", + default=flag_default("directory_hooks"), dest="directory_hooks", + help="Disable running executables found in Certbot's hook directories" + " during renewal. (default: False)") + helpful.add( + "renew", "--disable-renew-updates", action="store_true", + default=flag_default("disable_renew_updates"), dest="disable_renew_updates", + help="Disable automatic updates to your server configuration that" + " would otherwise be done by the selected installer plugin, and triggered" + " when the user executes \"certbot renew\", regardless of if the certificate" + " is renewed. This setting does not apply to important TLS configuration" + " updates.") + helpful.add( + "renew", "--no-autorenew", action="store_false", + default=flag_default("autorenew"), dest="autorenew", + help="Disable auto renewal of certificates.") + + # Populate the command line parameters for new style enhancements + enhancements.populate_cli(helpful.add) + + _create_subparsers(helpful) + _paths_parser(helpful) + # _plugins_parsing should be the last thing to act upon the main + # parser (--help should display plugin-specific options last) + _plugins_parsing(helpful, plugins) + + if not detect_defaults: + global helpful_parser # pylint: disable=global-statement + helpful_parser = helpful + return helpful.parse_args() + + +def set_by_cli(var): + """ + Return True if a particular config variable has been set by the user + (CLI or config file) including if the user explicitly set it to the + default. Returns False if the variable was assigned a default value. + """ + detector = set_by_cli.detector # type: ignore + if detector is None and helpful_parser is not None: + # Setup on first run: `detector` is a weird version of config in which + # the default value of every attribute is wrangled to be boolean-false + plugins = plugins_disco.PluginsRegistry.find_all() + # reconstructed_args == sys.argv[1:], or whatever was passed to main() + reconstructed_args = helpful_parser.args + [helpful_parser.verb] + detector = set_by_cli.detector = prepare_and_parse_args( # type: ignore + plugins, reconstructed_args, detect_defaults=True) + # propagate plugin requests: eg --standalone modifies config.authenticator + detector.authenticator, detector.installer = ( # type: ignore + plugin_selection.cli_plugin_requests(detector)) + + if not isinstance(getattr(detector, var), _Default): + logger.debug("Var %s=%s (set by user).", var, getattr(detector, var)) + return True + + for modifier in VAR_MODIFIERS.get(var, []): + if set_by_cli(modifier): + logger.debug("Var %s=%s (set by user).", + var, VAR_MODIFIERS.get(var, [])) + return True + + return False + + +# static housekeeping var +# functions attributed are not supported by mypy +# https://github.com/python/mypy/issues/2087 +set_by_cli.detector = None # type: ignore + + +def has_default_value(option, value): + """Does option have the default value? + + If the default value of option is not known, False is returned. + + :param str option: configuration variable being considered + :param value: value of the configuration variable named option + + :returns: True if option has the default value, otherwise, False + :rtype: bool + + """ + if helpful_parser is not None: + return (option in helpful_parser.defaults and + helpful_parser.defaults[option] == value) + return False + + +def option_was_set(option, value): + """Was option set by the user or does it differ from the default? + + :param str option: configuration variable being considered + :param value: value of the configuration variable named option + + :returns: True if the option was set, otherwise, False + :rtype: bool + + """ + return set_by_cli(option) or not has_default_value(option, value) + + +def argparse_type(variable): + """Return our argparse type function for a config variable (default: str)""" + # pylint: disable=protected-access + if helpful_parser is not None: + for action in helpful_parser.parser._actions: + if action.type is not None and action.dest == variable: + return action.type + return str diff --git a/certbot/certbot/_internal/cli/cli_constants.py b/certbot/certbot/_internal/cli/cli_constants.py new file mode 100644 index 000000000..748ae0d94 --- /dev/null +++ b/certbot/certbot/_internal/cli/cli_constants.py @@ -0,0 +1,107 @@ +"""Certbot command line constants""" +import sys + +from certbot.compat import os + +# For help strings, figure out how the user ran us. +# When invoked from letsencrypt-auto, sys.argv[0] is something like: +# "/home/user/.local/share/certbot/bin/certbot" +# Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before +# running letsencrypt-auto (and sudo stops us from seeing if they did), so it +# should only be used for purposes where inability to detect letsencrypt-auto +# fails safely + +LEAUTO = "letsencrypt-auto" +if "CERTBOT_AUTO" in os.environ: + # if we're here, this is probably going to be certbot-auto, unless the + # user saved the script under a different name + LEAUTO = os.path.basename(os.environ["CERTBOT_AUTO"]) + +old_path_fragment = os.path.join(".local", "share", "letsencrypt") +new_path_prefix = os.path.abspath(os.path.join(os.sep, "opt", + "eff.org", "certbot", "venv")) +if old_path_fragment in sys.argv[0] or sys.argv[0].startswith(new_path_prefix): + cli_command = LEAUTO +else: + cli_command = "certbot" + + +# Argparse's help formatting has a lot of unhelpful peculiarities, so we want +# to replace as much of it as we can... + +# This is the stub to include in help generated by argparse +SHORT_USAGE = """ + {0} [SUBCOMMAND] [options] [-d DOMAIN] [-d DOMAIN] ... + +Certbot can obtain and install HTTPS/TLS/SSL certificates. By default, +it will attempt to use a webserver both for obtaining and installing the +certificate. """.format(cli_command) + +# This section is used for --help and --help all ; it needs information +# about installed plugins to be fully formatted +COMMAND_OVERVIEW = """The most common SUBCOMMANDS and flags are: + +obtain, install, and renew certificates: + (default) run Obtain & install a certificate in your current webserver + certonly Obtain or renew a certificate, but do not install it + renew Renew all previously obtained certificates that are near expiry + enhance Add security enhancements to your existing configuration + -d DOMAINS Comma-separated list of domains to obtain a certificate for + + %s + --standalone Run a standalone webserver for authentication + %s + --webroot Place files in a server's webroot folder for authentication + --manual Obtain certificates interactively, or using shell script hooks + + -n Run non-interactively + --test-cert Obtain a test certificate from a staging server + --dry-run Test "renew" or "certonly" without saving any certificates to disk + +manage certificates: + certificates Display information about certificates you have from Certbot + revoke Revoke a certificate (supply --cert-name or --cert-path) + delete Delete a certificate (supply --cert-name) + +manage your account: + register Create an ACME account + unregister Deactivate an ACME account + update_account Update an ACME account + --agree-tos Agree to the ACME server's Subscriber Agreement + -m EMAIL Email address for important account notifications +""" + +# This is the short help for certbot --help, where we disable argparse +# altogether +HELP_AND_VERSION_USAGE = """ +More detailed help: + + -h, --help [TOPIC] print this message, or detailed help on a topic; + the available TOPICS are: + + all, automation, commands, paths, security, testing, or any of the + subcommands or plugins (certonly, renew, install, register, nginx, + apache, standalone, webroot, etc.) + -h all print a detailed help page including all topics + --version print the version number +""" + +# These argparse parameters should be removed when detecting defaults. +ARGPARSE_PARAMS_TO_REMOVE = ("const", "nargs", "type",) + + +# These sets are used when to help detect options set by the user. +EXIT_ACTIONS = set(("help", "version",)) + + +ZERO_ARG_ACTIONS = set(("store_const", "store_true", + "store_false", "append_const", "count",)) + + +# Maps a config option to a set of config options that may have modified it. +# This dictionary is used recursively, so if A modifies B and B modifies C, +# it is determined that C was modified by the user if A was modified. +VAR_MODIFIERS = {"account": set(("server",)), + "renew_hook": set(("deploy_hook",)), + "server": set(("dry_run", "staging",)), + "webroot_map": set(("webroot_path",))} diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py new file mode 100644 index 000000000..a0ddce38f --- /dev/null +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -0,0 +1,239 @@ +"""Certbot command line util function""" +import argparse +import copy + +import zope.interface.interface # pylint: disable=unused-import + +from acme import challenges +from certbot import interfaces +from certbot import util +from certbot import errors +from certbot.compat import os +from certbot._internal import constants + + +class _Default(object): + """A class to use as a default to detect if a value is set by a user""" + + def __bool__(self): + return False + + def __eq__(self, other): + return isinstance(other, _Default) + + def __hash__(self): + return id(_Default) + + def __nonzero__(self): + return self.__bool__() + + +def read_file(filename, mode="rb"): + """Returns the given file's contents. + + :param str filename: path to file + :param str mode: open mode (see `open`) + + :returns: absolute path of filename and its contents + :rtype: tuple + + :raises argparse.ArgumentTypeError: File does not exist or is not readable. + + """ + try: + filename = os.path.abspath(filename) + with open(filename, mode) as the_file: + contents = the_file.read() + return filename, contents + except IOError as exc: + raise argparse.ArgumentTypeError(exc.strerror) + + +def flag_default(name): + """Default value for CLI flag.""" + # XXX: this is an internal housekeeping notion of defaults before + # argparse has been set up; it is not accurate for all flags. Call it + # with caution. Plugin defaults are missing, and some things are using + # defaults defined in this file, not in constants.py :( + return copy.deepcopy(constants.CLI_DEFAULTS[name]) + + +def config_help(name, hidden=False): + """Extract the help message for an `.IConfig` attribute.""" + if hidden: + return argparse.SUPPRESS + field = interfaces.IConfig.__getitem__(name) # type: zope.interface.interface.Attribute + return field.__doc__ + + +class HelpfulArgumentGroup(object): + """Emulates an argparse group for use with HelpfulArgumentParser. + + This class is used in the add_group method of HelpfulArgumentParser. + Command line arguments can be added to the group, but help + suppression and default detection is applied by + HelpfulArgumentParser when necessary. + + """ + def __init__(self, helpful_arg_parser, topic): + self._parser = helpful_arg_parser + self._topic = topic + + def add_argument(self, *args, **kwargs): + """Add a new command line argument to the argument group.""" + self._parser.add(self._topic, *args, **kwargs) + + +class CustomHelpFormatter(argparse.HelpFormatter): + """This is a clone of ArgumentDefaultsHelpFormatter, with bugfixes. + + In particular we fix https://bugs.python.org/issue28742 + """ + + def _get_help_string(self, action): + helpstr = action.help + if '%(default)' not in action.help and '(default:' not in action.help: + if action.default != argparse.SUPPRESS: + defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + helpstr += ' (default: %(default)s)' + return helpstr + + +class _DomainsAction(argparse.Action): + """Action class for parsing domains.""" + + def __call__(self, parser, namespace, domain, option_string=None): + """Just wrap add_domains in argparseese.""" + add_domains(namespace, domain) + + +def add_domains(args_or_config, domains): + """Registers new domains to be used during the current client run. + + Domains are not added to the list of requested domains if they have + already been registered. + + :param args_or_config: parsed command line arguments + :type args_or_config: argparse.Namespace or + configuration.NamespaceConfig + :param str domain: one or more comma separated domains + + :returns: domains after they have been normalized and validated + :rtype: `list` of `str` + + """ + validated_domains = [] + for domain in domains.split(","): + domain = util.enforce_domain_sanity(domain.strip()) + validated_domains.append(domain) + if domain not in args_or_config.domains: + args_or_config.domains.append(domain) + + return validated_domains + + +class CaseInsensitiveList(list): + """A list that will ignore case when searching. + + This class is passed to the `choices` argument of `argparse.add_arguments` + through the `helpful` wrapper. It is necessary due to special handling of + command line arguments by `set_by_cli` in which the `type_func` is not applied.""" + def __contains__(self, element): + return super(CaseInsensitiveList, self).__contains__(element.lower()) + + +def _user_agent_comment_type(value): + if "(" in value or ")" in value: + raise argparse.ArgumentTypeError("may not contain parentheses") + return value + + +class _EncodeReasonAction(argparse.Action): + """Action class for parsing revocation reason.""" + + def __call__(self, parser, namespace, reason, option_string=None): + """Encodes the reason for certificate revocation.""" + code = constants.REVOCATION_REASONS[reason.lower()] + setattr(namespace, self.dest, code) + + +def parse_preferred_challenges(pref_challs): + """Translate and validate preferred challenges. + + :param pref_challs: list of preferred challenge types + :type pref_challs: `list` of `str` + + :returns: validated list of preferred challenge types + :rtype: `list` of `str` + + :raises errors.Error: if pref_challs is invalid + + """ + aliases = {"dns": "dns-01", "http": "http-01"} + challs = [c.strip() for c in pref_challs] + challs = [aliases.get(c, c) for c in challs] + + unrecognized = ", ".join(name for name in challs + if name not in challenges.Challenge.TYPES) + if unrecognized: + raise errors.Error( + "Unrecognized challenges: {0}".format(unrecognized)) + return challs + + +class _PrefChallAction(argparse.Action): + """Action class for parsing preferred challenges.""" + + def __call__(self, parser, namespace, pref_challs, option_string=None): + try: + challs = parse_preferred_challenges(pref_challs.split(",")) + except errors.Error as error: + raise argparse.ArgumentError(self, str(error)) + namespace.pref_challs.extend(challs) + + +class _DeployHookAction(argparse.Action): + """Action class for parsing deploy hooks.""" + + def __call__(self, parser, namespace, values, option_string=None): + renew_hook_set = namespace.deploy_hook != namespace.renew_hook + if renew_hook_set and namespace.renew_hook != values: + raise argparse.ArgumentError( + self, "conflicts with --renew-hook value") + namespace.deploy_hook = namespace.renew_hook = values + + +class _RenewHookAction(argparse.Action): + """Action class for parsing renew hooks.""" + + def __call__(self, parser, namespace, values, option_string=None): + deploy_hook_set = namespace.deploy_hook is not None + if deploy_hook_set and namespace.deploy_hook != values: + raise argparse.ArgumentError( + self, "conflicts with --deploy-hook value") + namespace.renew_hook = values + + +def nonnegative_int(value): + """Converts value to an int and checks that it is not negative. + + This function should used as the type parameter for argparse + arguments. + + :param str value: value provided on the command line + + :returns: integer representation of value + :rtype: int + + :raises argparse.ArgumentTypeError: if value isn't a non-negative integer + + """ + try: + int_value = int(value) + except ValueError: + raise argparse.ArgumentTypeError("value must be an integer") + + if int_value < 0: + raise argparse.ArgumentTypeError("value must be non-negative") + return int_value diff --git a/certbot/certbot/_internal/cli/group_adder.py b/certbot/certbot/_internal/cli/group_adder.py new file mode 100644 index 000000000..f22fbc496 --- /dev/null +++ b/certbot/certbot/_internal/cli/group_adder.py @@ -0,0 +1,19 @@ +"""This module contains a function to add the groups of arguments for the help +display""" +from certbot._internal.cli import VERB_HELP + + +def _add_all_groups(helpful): + helpful.add_group("automation", description="Flags for automating execution & other tweaks") + helpful.add_group("security", description="Security parameters & server settings") + helpful.add_group("testing", + description="The following flags are meant for testing and integration purposes only.") + helpful.add_group("paths", description="Flags for changing execution paths & servers") + helpful.add_group("manage", + description="Various subcommands and flags are available for managing your certificates:", + verbs=["certificates", "delete", "renew", "revoke", "update_symlinks"]) + + # VERBS + for verb, docs in VERB_HELP: + name = docs.get("realname", verb) + helpful.add_group(name, description=docs["opts"]) diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py new file mode 100644 index 000000000..e63ab4b87 --- /dev/null +++ b/certbot/certbot/_internal/cli/helpful.py @@ -0,0 +1,468 @@ +"""Certbot command line argument parser""" +from __future__ import print_function +import argparse +import copy +import glob +import sys +import configargparse +import six +import zope.component +import zope.interface + +from zope.interface import interfaces as zope_interfaces + +# pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Any, Dict, Optional +# pylint: enable=unused-import, no-name-in-module + +from certbot import crypto_util +from certbot import errors +from certbot import interfaces +from certbot import util +from certbot.compat import os +from certbot._internal import constants +from certbot._internal import hooks + +from certbot.display import util as display_util + +from certbot._internal.cli import ( + SHORT_USAGE, + CustomHelpFormatter, + flag_default, + VERB_HELP, + VERB_HELP_MAP, + COMMAND_OVERVIEW, + HELP_AND_VERSION_USAGE, + _Default, + add_domains, + EXIT_ACTIONS, + ZERO_ARG_ACTIONS, + ARGPARSE_PARAMS_TO_REMOVE, + HelpfulArgumentGroup +) + + +class HelpfulArgumentParser(object): + """Argparse Wrapper. + + This class wraps argparse, adding the ability to make --help less + verbose, and request help on specific subcategories at a time, eg + 'certbot --help security' for security options. + + """ + def __init__(self, args, plugins, detect_defaults=False): + from certbot._internal import main + self.VERBS = { + "auth": main.certonly, + "certonly": main.certonly, + "run": main.run, + "install": main.install, + "plugins": main.plugins_cmd, + "register": main.register, + "update_account": main.update_account, + "unregister": main.unregister, + "renew": main.renew, + "revoke": main.revoke, + "rollback": main.rollback, + "everything": main.run, + "update_symlinks": main.update_symlinks, + "certificates": main.certificates, + "delete": main.delete, + "enhance": main.enhance, + } + + # Get notification function for printing + try: + self.notify = zope.component.getUtility( + interfaces.IDisplay).notification + except zope_interfaces.ComponentLookupError: + self.notify = display_util.NoninteractiveDisplay( + sys.stdout).notification + + + # List of topics for which additional help can be provided + HELP_TOPICS = ["all", "security", "paths", "automation", "testing"] + HELP_TOPICS += list(self.VERBS) + self.COMMANDS_TOPICS + ["manage"] + + plugin_names = list(plugins) + self.help_topics = HELP_TOPICS + plugin_names + [None] # type: ignore + + self.detect_defaults = detect_defaults + self.args = args + + if self.args and self.args[0] == 'help': + self.args[0] = '--help' + + self.determine_verb() + help1 = self.prescan_for_flag("-h", self.help_topics) + help2 = self.prescan_for_flag("--help", self.help_topics) + if isinstance(help1, bool) and isinstance(help2, bool): + self.help_arg = help1 or help2 + else: + self.help_arg = help1 if isinstance(help1, six.string_types) else help2 + + short_usage = self._usage_string(plugins, self.help_arg) + + self.visible_topics = self.determine_help_topics(self.help_arg) + + # elements are added by .add_group() + self.groups = {} # type: Dict[str, argparse._ArgumentGroup] + # elements are added by .parse_args() + self.defaults = {} # type: Dict[str, Any] + + self.parser = configargparse.ArgParser( + prog="certbot", + usage=short_usage, + formatter_class=CustomHelpFormatter, + args_for_setting_config_path=["-c", "--config"], + default_config_files=flag_default("config_files"), + config_arg_help_message="path to config file (default: {0})".format( + " and ".join(flag_default("config_files")))) + + # This is the only way to turn off overly verbose config flag documentation + self.parser._add_config_file_help = False + + # Help that are synonyms for --help subcommands + COMMANDS_TOPICS = ["command", "commands", "subcommand", "subcommands", "verbs"] + + def _list_subcommands(self): + longest = max(len(v) for v in VERB_HELP_MAP) + + text = "The full list of available SUBCOMMANDS is:\n\n" + for verb, props in sorted(VERB_HELP): + doc = props.get("short", "") + text += '{0:<{length}} {1}\n'.format(verb, doc, length=longest) + + text += "\nYou can get more help on a specific subcommand with --help SUBCOMMAND\n" + return text + + def _usage_string(self, plugins, help_arg): + """Make usage strings late so that plugins can be initialised late + + :param plugins: all discovered plugins + :param help_arg: False for none; True for --help; "TOPIC" for --help TOPIC + :rtype: str + :returns: a short usage string for the top of --help TOPIC) + """ + if "nginx" in plugins: + nginx_doc = "--nginx Use the Nginx plugin for authentication & installation" + else: + nginx_doc = "(the certbot nginx plugin is not installed)" + if "apache" in plugins: + apache_doc = "--apache Use the Apache plugin for authentication & installation" + else: + apache_doc = "(the certbot apache plugin is not installed)" + + usage = SHORT_USAGE + if help_arg is True: + self.notify(usage + COMMAND_OVERVIEW % (apache_doc, nginx_doc) + HELP_AND_VERSION_USAGE) + sys.exit(0) + elif help_arg in self.COMMANDS_TOPICS: + self.notify(usage + self._list_subcommands()) + sys.exit(0) + elif help_arg == "all": + # if we're doing --help all, the OVERVIEW is part of the SHORT_USAGE at + # the top; if we're doing --help someothertopic, it's OT so it's not + usage += COMMAND_OVERVIEW % (apache_doc, nginx_doc) + else: + custom = VERB_HELP_MAP.get(help_arg, {}).get("usage", None) + usage = custom if custom else usage + + return usage + + def remove_config_file_domains_for_renewal(self, parsed_args): + """Make "certbot renew" safe if domains are set in cli.ini.""" + # Works around https://github.com/certbot/certbot/issues/4096 + if self.verb == "renew": + for source, flags in self.parser._source_to_settings.items(): # pylint: disable=protected-access + if source.startswith("config_file") and "domains" in flags: + parsed_args.domains = _Default() if self.detect_defaults else [] + + def parse_args(self): + """Parses command line arguments and returns the result. + + :returns: parsed command line arguments + :rtype: argparse.Namespace + + """ + parsed_args = self.parser.parse_args(self.args) + parsed_args.func = self.VERBS[self.verb] + parsed_args.verb = self.verb + + self.remove_config_file_domains_for_renewal(parsed_args) + + if self.detect_defaults: + return parsed_args + + self.defaults = dict((key, copy.deepcopy(self.parser.get_default(key))) + for key in vars(parsed_args)) + + # Do any post-parsing homework here + + if self.verb == "renew": + if parsed_args.force_interactive: + raise errors.Error( + "{0} cannot be used with renew".format( + constants.FORCE_INTERACTIVE_FLAG)) + parsed_args.noninteractive_mode = True + + if parsed_args.force_interactive and parsed_args.noninteractive_mode: + raise errors.Error( + "Flag for non-interactive mode and {0} conflict".format( + constants.FORCE_INTERACTIVE_FLAG)) + + if parsed_args.staging or parsed_args.dry_run: + self.set_test_server(parsed_args) + + if parsed_args.csr: + self.handle_csr(parsed_args) + + if parsed_args.must_staple: + parsed_args.staple = True + + if parsed_args.validate_hooks: + hooks.validate_hooks(parsed_args) + + if parsed_args.allow_subset_of_names: + if any(util.is_wildcard_domain(d) for d in parsed_args.domains): + raise errors.Error("Using --allow-subset-of-names with a" + " wildcard domain is not supported.") + + if parsed_args.hsts and parsed_args.auto_hsts: + raise errors.Error( + "Parameters --hsts and --auto-hsts cannot be used simultaneously.") + + return parsed_args + + def set_test_server(self, parsed_args): + """We have --staging/--dry-run; perform sanity check and set config.server""" + + # Flag combinations should produce these results: + # | --staging | --dry-run | + # ------------------------------------------------------------ + # | --server acme-v02 | Use staging | Use staging | + # | --server acme-staging-v02 | Use staging | Use staging | + # | --server | Conflict error | Use | + + default_servers = (flag_default("server"), constants.STAGING_URI) + + if parsed_args.staging and parsed_args.server not in default_servers: + raise errors.Error("--server value conflicts with --staging") + + if parsed_args.server in default_servers: + parsed_args.server = constants.STAGING_URI + + if parsed_args.dry_run: + if self.verb not in ["certonly", "renew"]: + raise errors.Error("--dry-run currently only works with the " + "'certonly' or 'renew' subcommands (%r)" % self.verb) + parsed_args.break_my_certs = parsed_args.staging = True + if glob.glob(os.path.join(parsed_args.config_dir, constants.ACCOUNTS_DIR, "*")): + # The user has a prod account, but might not have a staging + # one; we don't want to start trying to perform interactive registration + parsed_args.tos = True + parsed_args.register_unsafely_without_email = True + + def handle_csr(self, parsed_args): + """Process a --csr flag.""" + if parsed_args.verb != "certonly": + raise errors.Error("Currently, a CSR file may only be specified " + "when obtaining a new or replacement " + "via the certonly command. Please try the " + "certonly command instead.") + if parsed_args.allow_subset_of_names: + raise errors.Error("--allow-subset-of-names cannot be used with --csr") + + csrfile, contents = parsed_args.csr[0:2] + typ, csr, domains = crypto_util.import_csr_file(csrfile, contents) + + # This is not necessary for webroot to work, however, + # obtain_certificate_from_csr requires parsed_args.domains to be set + for domain in domains: + add_domains(parsed_args, domain) + + if not domains: + # TODO: add CN to domains instead: + raise errors.Error( + "Unfortunately, your CSR %s needs to have a SubjectAltName for every domain" + % parsed_args.csr[0]) + + parsed_args.actual_csr = (csr, typ) + + csr_domains = {d.lower() for d in domains} + config_domains = set(parsed_args.domains) + if csr_domains != config_domains: + raise errors.ConfigurationError( + "Inconsistent domain requests:\nFrom the CSR: {0}\nFrom command line/config: {1}" + .format(", ".join(csr_domains), ", ".join(config_domains))) + + + def determine_verb(self): + """Determines the verb/subcommand provided by the user. + + This function works around some of the limitations of argparse. + + """ + if "-h" in self.args or "--help" in self.args: + # all verbs double as help arguments; don't get them confused + self.verb = "help" + return + + for i, token in enumerate(self.args): + if token in self.VERBS: + verb = token + if verb == "auth": + verb = "certonly" + if verb == "everything": + verb = "run" + self.verb = verb + self.args.pop(i) + return + + self.verb = "run" + + def prescan_for_flag(self, flag, possible_arguments): + """Checks cli input for flags. + + Check for a flag, which accepts a fixed set of possible arguments, in + the command line; we will use this information to configure argparse's + help correctly. Return the flag's argument, if it has one that matches + the sequence @possible_arguments; otherwise return whether the flag is + present. + + """ + if flag not in self.args: + return False + pos = self.args.index(flag) + try: + nxt = self.args[pos + 1] + if nxt in possible_arguments: + return nxt + except IndexError: + pass + return True + + def add(self, topics, *args, **kwargs): + """Add a new command line argument. + + :param topics: str or [str] help topic(s) this should be listed under, + or None for options that don't fit under a specific + topic which will only be shown in "--help all" output. + The first entry determines where the flag lives in the + "--help all" output (None -> "optional arguments"). + :param list *args: the names of this argument flag + :param dict **kwargs: various argparse settings for this argument + + """ + + if isinstance(topics, list): + # if this flag can be listed in multiple sections, try to pick the one + # that the user has asked for help about + topic = self.help_arg if self.help_arg in topics else topics[0] + else: + topic = topics # there's only one + + if self.detect_defaults: + kwargs = self.modify_kwargs_for_default_detection(**kwargs) + + if self.visible_topics[topic]: + if topic in self.groups: + group = self.groups[topic] + group.add_argument(*args, **kwargs) + else: + self.parser.add_argument(*args, **kwargs) + else: + kwargs["help"] = argparse.SUPPRESS + self.parser.add_argument(*args, **kwargs) + + def modify_kwargs_for_default_detection(self, **kwargs): + """Modify an arg so we can check if it was set by the user. + + Changes the parameters given to argparse when adding an argument + so we can properly detect if the value was set by the user. + + :param dict kwargs: various argparse settings for this argument + + :returns: a modified versions of kwargs + :rtype: dict + + """ + action = kwargs.get("action", None) + if action not in EXIT_ACTIONS: + kwargs["action"] = ("store_true" if action in ZERO_ARG_ACTIONS else + "store") + kwargs["default"] = _Default() + for param in ARGPARSE_PARAMS_TO_REMOVE: + kwargs.pop(param, None) + + return kwargs + + def add_deprecated_argument(self, argument_name, num_args): + """Adds a deprecated argument with the name argument_name. + + Deprecated arguments are not shown in the help. If they are used + on the command line, a warning is shown stating that the + argument is deprecated and no other action is taken. + + :param str argument_name: Name of deprecated argument. + :param int nargs: Number of arguments the option takes. + + """ + util.add_deprecated_argument( + self.parser.add_argument, argument_name, num_args) + + def add_group(self, topic, verbs=(), **kwargs): + """Create a new argument group. + + This method must be called once for every topic, however, calls + to this function are left next to the argument definitions for + clarity. + + :param str topic: Name of the new argument group. + :param str verbs: List of subcommands that should be documented as part of + this help group / topic + + :returns: The new argument group. + :rtype: `HelpfulArgumentGroup` + + """ + if self.visible_topics[topic]: + self.groups[topic] = self.parser.add_argument_group(topic, **kwargs) + if self.help_arg: + for v in verbs: + self.groups[topic].add_argument(v, help=VERB_HELP_MAP[v]["short"]) + return HelpfulArgumentGroup(self, topic) + + def add_plugin_args(self, plugins): + """ + + Let each of the plugins add its own command line arguments, which + may or may not be displayed as help topics. + + """ + for name, plugin_ep in six.iteritems(plugins): + parser_or_group = self.add_group(name, + description=plugin_ep.long_description) + plugin_ep.plugin_cls.inject_parser_options(parser_or_group, name) + + def determine_help_topics(self, chosen_topic): + """ + + The user may have requested help on a topic, return a dict of which + topics to display. @chosen_topic has prescan_for_flag's return type + + :returns: dict + + """ + # topics maps each topic to whether it should be documented by + # argparse on the command line + if chosen_topic == "auth": + chosen_topic = "certonly" + if chosen_topic == "everything": + chosen_topic = "run" + if chosen_topic == "all": + # Addition of condition closes #6209 (removal of duplicate route53 option). + return {t: t != 'certbot-route53:auth' for t in self.help_topics} + elif not chosen_topic: + return {t: False for t in self.help_topics} + return {t: t == chosen_topic for t in self.help_topics} diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py new file mode 100644 index 000000000..4378435d7 --- /dev/null +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -0,0 +1,50 @@ +"""This is a module that adds configuration to the argument parser regarding +paths for certificates""" +from certbot.compat import os +from certbot._internal.cli import ( + read_file, + flag_default, + config_help +) + + +def _paths_parser(helpful): + add = helpful.add + verb = helpful.verb + if verb == "help": + verb = helpful.help_arg + + cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked." + sections = ["paths", "install", "revoke", "certonly", "manage"] + if verb == "certonly": + add(sections, "--cert-path", type=os.path.abspath, + default=flag_default("auth_cert_path"), help=cph) + elif verb == "revoke": + add(sections, "--cert-path", type=read_file, required=False, help=cph) + else: + add(sections, "--cert-path", type=os.path.abspath, help=cph) + + section = "paths" + if verb in ("install", "revoke"): + section = verb + # revoke --key-path reads a file, install --key-path takes a string + add(section, "--key-path", + type=((verb == "revoke" and read_file) or os.path.abspath), + help="Path to private key for certificate installation " + "or revocation (if account key is missing)") + + default_cp = None + if verb == "certonly": + default_cp = flag_default("auth_chain_path") + add(["paths", "install"], "--fullchain-path", default=default_cp, type=os.path.abspath, + help="Accompanying path to a full certificate chain (certificate plus chain).") + add("paths", "--chain-path", default=default_cp, type=os.path.abspath, + help="Accompanying path to a certificate chain.") + add("paths", "--config-dir", default=flag_default("config_dir"), + help=config_help("config_dir")) + add("paths", "--work-dir", default=flag_default("work_dir"), + help=config_help("work_dir")) + add("paths", "--logs-dir", default=flag_default("logs_dir"), + help="Logs directory.") + add("paths", "--server", default=flag_default("server"), + help=config_help("server")) diff --git a/certbot/certbot/_internal/cli/plugins_parsing.py b/certbot/certbot/_internal/cli/plugins_parsing.py new file mode 100644 index 000000000..9e11ad3ab --- /dev/null +++ b/certbot/certbot/_internal/cli/plugins_parsing.py @@ -0,0 +1,97 @@ +"""This is a module that handles parsing of plugins for the argument parser""" +from certbot._internal.cli import flag_default + + +def _plugins_parsing(helpful, plugins): + # It's nuts, but there are two "plugins" topics. Somehow this works + helpful.add_group( + "plugins", description="Plugin Selection: Certbot client supports an " + "extensible plugins architecture. See '%(prog)s plugins' for a " + "list of all installed plugins and their names. You can force " + "a particular plugin by setting options provided below. Running " + "--help will list flags specific to that plugin.") + + helpful.add("plugins", "--configurator", default=flag_default("configurator"), + help="Name of the plugin that is both an authenticator and an installer." + " Should not be used together with --authenticator or --installer. " + "(default: Ask)") + helpful.add("plugins", "-a", "--authenticator", default=flag_default("authenticator"), + help="Authenticator plugin name.") + helpful.add("plugins", "-i", "--installer", default=flag_default("installer"), + help="Installer plugin name (also used to find domains).") + helpful.add(["plugins", "certonly", "run", "install"], + "--apache", action="store_true", default=flag_default("apache"), + help="Obtain and install certificates using Apache") + helpful.add(["plugins", "certonly", "run", "install"], + "--nginx", action="store_true", default=flag_default("nginx"), + help="Obtain and install certificates using Nginx") + helpful.add(["plugins", "certonly"], "--standalone", action="store_true", + default=flag_default("standalone"), + help='Obtain certificates using a "standalone" webserver.') + helpful.add(["plugins", "certonly"], "--manual", action="store_true", + default=flag_default("manual"), + help="Provide laborious manual instructions for obtaining a certificate") + helpful.add(["plugins", "certonly"], "--webroot", action="store_true", + default=flag_default("webroot"), + help="Obtain certificates by placing files in a webroot directory.") + helpful.add(["plugins", "certonly"], "--dns-cloudflare", action="store_true", + default=flag_default("dns_cloudflare"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using Cloudflare for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-cloudxns", action="store_true", + default=flag_default("dns_cloudxns"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using CloudXNS for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-digitalocean", action="store_true", + default=flag_default("dns_digitalocean"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using DigitalOcean for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-dnsimple", action="store_true", + default=flag_default("dns_dnsimple"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using DNSimple for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-dnsmadeeasy", action="store_true", + default=flag_default("dns_dnsmadeeasy"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using DNS Made Easy for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-gehirn", action="store_true", + default=flag_default("dns_gehirn"), + help=("Obtain certificates using a DNS TXT record " + "(if you are using Gehirn Infrastructure Service for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-google", action="store_true", + default=flag_default("dns_google"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using Google Cloud DNS).")) + helpful.add(["plugins", "certonly"], "--dns-linode", action="store_true", + default=flag_default("dns_linode"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using Linode for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-luadns", action="store_true", + default=flag_default("dns_luadns"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using LuaDNS for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-nsone", action="store_true", + default=flag_default("dns_nsone"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using NS1 for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-ovh", action="store_true", + default=flag_default("dns_ovh"), + help=("Obtain certificates using a DNS TXT record (if you are " + "using OVH for DNS).")) + helpful.add(["plugins", "certonly"], "--dns-rfc2136", action="store_true", + default=flag_default("dns_rfc2136"), + help="Obtain certificates using a DNS TXT record (if you are using BIND for DNS).") + helpful.add(["plugins", "certonly"], "--dns-route53", action="store_true", + default=flag_default("dns_route53"), + help=("Obtain certificates using a DNS TXT record (if you are using Route53 for " + "DNS).")) + helpful.add(["plugins", "certonly"], "--dns-sakuracloud", action="store_true", + default=flag_default("dns_sakuracloud"), + help=("Obtain certificates using a DNS TXT record " + "(if you are using Sakura Cloud for DNS).")) + + # things should not be reorder past/pre this comment: + # plugins_group should be displayed in --help before plugin + # specific groups (so that plugins_group.description makes sense) + + helpful.add_plugin_args(plugins) diff --git a/certbot/certbot/_internal/cli/report_config_interaction.py b/certbot/certbot/_internal/cli/report_config_interaction.py new file mode 100644 index 000000000..39266e776 --- /dev/null +++ b/certbot/certbot/_internal/cli/report_config_interaction.py @@ -0,0 +1,27 @@ +"""This is a module that reports config option interaction that should be +checked by set_by_cli""" +import six + +from certbot._internal.cli import VAR_MODIFIERS + + +def report_config_interaction(modified, modifiers): + """Registers config option interaction to be checked by set_by_cli. + + This function can be called by during the __init__ or + add_parser_arguments methods of plugins to register interactions + between config options. + + :param modified: config options that can be modified by modifiers + :type modified: iterable or str (string_types) + :param modifiers: config options that modify modified + :type modifiers: iterable or str (string_types) + + """ + if isinstance(modified, six.string_types): + modified = (modified,) + if isinstance(modifiers, six.string_types): + modifiers = (modifiers,) + + for var in modified: + VAR_MODIFIERS.setdefault(var, set()).update(modifiers) diff --git a/certbot/certbot/_internal/cli/subparsers.py b/certbot/certbot/_internal/cli/subparsers.py new file mode 100644 index 000000000..13f8705ce --- /dev/null +++ b/certbot/certbot/_internal/cli/subparsers.py @@ -0,0 +1,72 @@ +"""This module creates subparsers for the argument parser""" +from certbot import interfaces +from certbot._internal import constants + +from certbot._internal.cli import ( + flag_default, + read_file, + CaseInsensitiveList, + _user_agent_comment_type, + _EncodeReasonAction +) + + +def _create_subparsers(helpful): + from certbot._internal.client import sample_user_agent # avoid import loops + helpful.add( + None, "--user-agent", default=flag_default("user_agent"), + help='Set a custom user agent string for the client. User agent strings allow ' + 'the CA to collect high level statistics about success rates by OS, ' + 'plugin and use case, and to know when to deprecate support for past Python ' + "versions and flags. If you wish to hide this information from the Let's " + 'Encrypt server, set this to "". ' + '(default: {0}). The flags encoded in the user agent are: ' + '--duplicate, --force-renew, --allow-subset-of-names, -n, and ' + 'whether any hooks are set.'.format(sample_user_agent())) + helpful.add( + None, "--user-agent-comment", default=flag_default("user_agent_comment"), + type=_user_agent_comment_type, + help="Add a comment to the default user agent string. May be used when repackaging Certbot " + "or calling it from another tool to allow additional statistical data to be collected." + " Ignored if --user-agent is set. (Example: Foo-Wrapper/1.0)") + helpful.add("certonly", + "--csr", default=flag_default("csr"), type=read_file, + help="Path to a Certificate Signing Request (CSR) in DER or PEM format." + " Currently --csr only works with the 'certonly' subcommand.") + helpful.add("revoke", + "--reason", dest="reason", + choices=CaseInsensitiveList(sorted(constants.REVOCATION_REASONS, + key=constants.REVOCATION_REASONS.get)), + action=_EncodeReasonAction, default=flag_default("reason"), + help="Specify reason for revoking certificate. (default: unspecified)") + helpful.add("revoke", + "--delete-after-revoke", action="store_true", + default=flag_default("delete_after_revoke"), + help="Delete certificates after revoking them, along with all previous and later " + "versions of those certificates.") + helpful.add("revoke", + "--no-delete-after-revoke", action="store_false", + dest="delete_after_revoke", + default=flag_default("delete_after_revoke"), + help="Do not delete certificates after revoking them. This " + "option should be used with caution because the 'renew' " + "subcommand will attempt to renew undeleted revoked " + "certificates.") + helpful.add("rollback", + "--checkpoints", type=int, metavar="N", + default=flag_default("rollback_checkpoints"), + help="Revert configuration N number of checkpoints.") + helpful.add("plugins", + "--init", action="store_true", default=flag_default("init"), + help="Initialize plugins.") + helpful.add("plugins", + "--prepare", action="store_true", default=flag_default("prepare"), + help="Initialize and prepare plugins.") + helpful.add("plugins", + "--authenticators", action="append_const", dest="ifaces", + default=flag_default("ifaces"), + const=interfaces.IAuthenticator, help="Limit to authenticator plugins only.") + helpful.add("plugins", + "--installers", action="append_const", dest="ifaces", + default=flag_default("ifaces"), + const=interfaces.IInstaller, help="Limit to installer plugins only.") diff --git a/certbot/certbot/_internal/cli/verb_help.py b/certbot/certbot/_internal/cli/verb_help.py new file mode 100644 index 000000000..131cfec96 --- /dev/null +++ b/certbot/certbot/_internal/cli/verb_help.py @@ -0,0 +1,106 @@ +"""This module contain help information for verbs supported by certbot""" +from certbot.compat import os +from certbot._internal.cli import ( + SHORT_USAGE, + flag_default +) + +# The attributes here are: +# short: a string that will be displayed by "certbot -h commands" +# opts: a string that heads the section of flags with which this command is documented, +# both for "certbot -h SUBCOMMAND" and "certbot -h all" +# usage: an optional string that overrides the header of "certbot -h SUBCOMMAND" +VERB_HELP = [ + ("run (default)", { + "short": "Obtain/renew a certificate, and install it", + "opts": "Options for obtaining & installing certificates", + "usage": SHORT_USAGE.replace("[SUBCOMMAND]", ""), + "realname": "run" + }), + ("certonly", { + "short": "Obtain or renew a certificate, but do not install it", + "opts": "Options for modifying how a certificate is obtained", + "usage": ("\n\n certbot certonly [options] [-d DOMAIN] [-d DOMAIN] ...\n\n" + "This command obtains a TLS/SSL certificate without installing it anywhere.") + }), + ("renew", { + "short": "Renew all certificates (or one specified with --cert-name)", + "opts": ("The 'renew' subcommand will attempt to renew all" + " certificates (or more precisely, certificate lineages) you have" + " previously obtained if they are close to expiry, and print a" + " summary of the results. By default, 'renew' will reuse the options" + " used to create obtain or most recently successfully renew each" + " certificate lineage. You can try it with `--dry-run` first. For" + " more fine-grained control, you can renew individual lineages with" + " the `certonly` subcommand. Hooks are available to run commands" + " before and after renewal; see" + " https://certbot.eff.org/docs/using.html#renewal for more" + " information on these."), + "usage": "\n\n certbot renew [--cert-name CERTNAME] [options]\n\n" + }), + ("certificates", { + "short": "List certificates managed by Certbot", + "opts": "List certificates managed by Certbot", + "usage": ("\n\n certbot certificates [options] ...\n\n" + "Print information about the status of certificates managed by Certbot.") + }), + ("delete", { + "short": "Clean up all files related to a certificate", + "opts": "Options for deleting a certificate", + "usage": "\n\n certbot delete --cert-name CERTNAME\n\n" + }), + ("revoke", { + "short": "Revoke a certificate specified with --cert-path or --cert-name", + "opts": "Options for revocation of certificates", + "usage": "\n\n certbot revoke [--cert-path /path/to/fullchain.pem | " + "--cert-name example.com] [options]\n\n" + }), + ("register", { + "short": "Register for account with Let's Encrypt / other ACME server", + "opts": "Options for account registration", + "usage": "\n\n certbot register --email user@example.com [options]\n\n" + }), + ("update_account", { + "short": "Update existing account with Let's Encrypt / other ACME server", + "opts": "Options for account modification", + "usage": "\n\n certbot update_account --email updated_email@example.com [options]\n\n" + }), + ("unregister", { + "short": "Irrevocably deactivate your account", + "opts": "Options for account deactivation.", + "usage": "\n\n certbot unregister [options]\n\n" + }), + ("install", { + "short": "Install an arbitrary certificate in a server", + "opts": "Options for modifying how a certificate is deployed", + "usage": "\n\n certbot install --cert-path /path/to/fullchain.pem " + " --key-path /path/to/private-key [options]\n\n" + }), + ("rollback", { + "short": "Roll back server conf changes made during certificate installation", + "opts": "Options for rolling back server configuration changes", + "usage": "\n\n certbot rollback --checkpoints 3 [options]\n\n" + }), + ("plugins", { + "short": "List plugins that are installed and available on your system", + "opts": 'Options for the "plugins" subcommand', + "usage": "\n\n certbot plugins [options]\n\n" + }), + ("update_symlinks", { + "short": "Recreate symlinks in your /etc/letsencrypt/live/ directory", + "opts": ("Recreates certificate and key symlinks in {0}, if you changed them by hand " + "or edited a renewal configuration file".format( + os.path.join(flag_default("config_dir"), "live"))), + "usage": "\n\n certbot update_symlinks [options]\n\n" + }), + ("enhance", { + "short": "Add security enhancements to your existing configuration", + "opts": ("Helps to harden the TLS configuration by adding security enhancements " + "to already existing configuration."), + "usage": "\n\n certbot enhance [options]\n\n" + }), +] + + +# VERB_HELP is a list in order to preserve order, but a dict is sometimes useful +VERB_HELP_MAP = dict(VERB_HELP) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 05da1da4e..3a7fb57f8 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -93,7 +93,7 @@ class ParseTest(unittest.TestCase): return output.getvalue() - @mock.patch("certbot._internal.cli.flag_default") + @mock.patch("certbot._internal.cli.helpful.flag_default") def test_cli_ini_domains(self, mock_flag_default): with tempfile.NamedTemporaryFile() as tmp_config: tmp_config.close() # close now because of compatibility issues on Windows diff --git a/certbot/tests/helpful_test.py b/certbot/tests/helpful_test.py new file mode 100644 index 000000000..292e55304 --- /dev/null +++ b/certbot/tests/helpful_test.py @@ -0,0 +1,193 @@ +"""Tests for certbot.helpful_parser""" +import unittest + +from certbot import errors +from certbot._internal.cli import HelpfulArgumentParser +from certbot._internal.cli import _DomainsAction +from certbot._internal import constants + + +class TestScanningFlags(unittest.TestCase): + '''Test the prescan_for_flag method of HelpfulArgumentParser''' + def test_prescan_no_help_flag(self): + arg_parser = HelpfulArgumentParser(['run'], {}) + detected_flag = arg_parser.prescan_for_flag('--help', + ['all', 'certonly']) + self.assertFalse(detected_flag) + detected_flag = arg_parser.prescan_for_flag('-h', + ['all, certonly']) + self.assertFalse(detected_flag) + + def test_prescan_unvalid_topic(self): + arg_parser = HelpfulArgumentParser(['--help', 'all'], {}) + detected_flag = arg_parser.prescan_for_flag('--help', + ['potato']) + self.assertIs(detected_flag, True) + detected_flag = arg_parser.prescan_for_flag('-h', + arg_parser.help_topics) + self.assertFalse(detected_flag) + + def test_prescan_valid_topic(self): + arg_parser = HelpfulArgumentParser(['-h', 'all'], {}) + detected_flag = arg_parser.prescan_for_flag('-h', + arg_parser.help_topics) + self.assertEqual(detected_flag, 'all') + detected_flag = arg_parser.prescan_for_flag('--help', + arg_parser.help_topics) + self.assertFalse(detected_flag) + +class TestDetermineVerbs(unittest.TestCase): + '''Tests for determine_verb methods of HelpfulArgumentParser''' + def test_determine_verb_wrong_verb(self): + arg_parser = HelpfulArgumentParser(['potato'], {}) + self.assertEqual(arg_parser.verb, "run") + self.assertEqual(arg_parser.args, ["potato"]) + + def test_determine_verb_help(self): + arg_parser = HelpfulArgumentParser(['--help', 'everything'], {}) + self.assertEqual(arg_parser.verb, "help") + self.assertEqual(arg_parser.args, ["--help", "everything"]) + arg_parser = HelpfulArgumentParser(['-d', 'some_domain', '--help', + 'all'], {}) + self.assertEqual(arg_parser.verb, "help") + self.assertEqual(arg_parser.args, ['-d', 'some_domain', '--help', + 'all']) + + def test_determine_verb(self): + arg_parser = HelpfulArgumentParser(['certonly'], {}) + self.assertEqual(arg_parser.verb, 'certonly') + self.assertEqual(arg_parser.args, []) + + arg_parser = HelpfulArgumentParser(['auth'], {}) + self.assertEqual(arg_parser.verb, 'certonly') + self.assertEqual(arg_parser.args, []) + + arg_parser = HelpfulArgumentParser(['everything'], {}) + self.assertEqual(arg_parser.verb, 'run') + self.assertEqual(arg_parser.args, []) + + +class TestAdd(unittest.TestCase): + '''Tests for add method in HelpfulArgumentParser''' + def test_add_trivial_argument(self): + arg_parser = HelpfulArgumentParser(['run'], {}) + arg_parser.add(None, "--hello-world") + parsed_args = arg_parser.parser.parse_args(['--hello-world', + 'Hello World!']) + self.assertIs(parsed_args.hello_world, 'Hello World!') + self.assertFalse(hasattr(parsed_args, 'potato')) + + def test_add_expected_argument(self): + arg_parser = HelpfulArgumentParser(['--help', 'run'], {}) + arg_parser.add( + [None, "run", "certonly", "register"], + "--eab-kid", dest="eab_kid", action="store", + metavar="EAB_KID", + help="Key Identifier for External Account Binding") + parsed_args = arg_parser.parser.parse_args(["--eab-kid", None]) + self.assertIs(parsed_args.eab_kid, None) + self.assertTrue(hasattr(parsed_args, 'eab_kid')) + + +class TestAddGroup(unittest.TestCase): + '''Test add_group method of HelpfulArgumentParser''' + def test_add_group_no_input(self): + arg_parser = HelpfulArgumentParser(['run'], {}) + self.assertRaises(TypeError, arg_parser.add_group) + + def test_add_group_topic_not_visible(self): + # The user request help on run. A topic that given somewhere in the + # args won't be added to the groups in the parser. + arg_parser = HelpfulArgumentParser(['--help', 'run'], {}) + arg_parser.add_group("auth", + description="description of auth") + self.assertEqual(arg_parser.groups, {}) + + def test_add_group_topic_requested_help(self): + arg_parser = HelpfulArgumentParser(['--help', 'run'], {}) + arg_parser.add_group("run", + description="description of run") + self.assertTrue(arg_parser.groups["run"]) + arg_parser.add_group("certonly", description="description of certonly") + with self.assertRaises(KeyError): + self.assertFalse(arg_parser.groups["certonly"]) + + +class TestParseArgsErrors(unittest.TestCase): + '''Tests for errors that should be met for some cases in parse_args method + in HelpfulArgumentParser''' + def test_parse_args_renew_force_interactive(self): + arg_parser = HelpfulArgumentParser(['renew', '--force-interactive'], + {}) + arg_parser.add( + None, constants.FORCE_INTERACTIVE_FLAG, action="store_true") + + with self.assertRaises(errors.Error): + arg_parser.parse_args() + + def test_parse_args_non_interactive_and_force_interactive(self): + arg_parser = HelpfulArgumentParser(['--force-interactive', + '--non-interactive'], {}) + arg_parser.add( + None, constants.FORCE_INTERACTIVE_FLAG, action="store_true") + arg_parser.add( + None, "--non-interactive", dest="noninteractive_mode", + action="store_true" + ) + + with self.assertRaises(errors.Error): + arg_parser.parse_args() + + def test_parse_args_subset_names_wildcard_domain(self): + arg_parser = HelpfulArgumentParser(['--domain', + '*.example.com,potato.example.com', + '--allow-subset-of-names'], {}) + # The following arguments are added because they have to be defined + # in order for arg_parser to run completely. They are not used for the + # test. + arg_parser.add( + None, constants.FORCE_INTERACTIVE_FLAG, action="store_true") + arg_parser.add( + None, "--non-interactive", dest="noninteractive_mode", + action="store_true") + arg_parser.add( + None, "--staging" + ) + arg_parser.add(None, "--dry-run") + arg_parser.add(None, "--csr") + arg_parser.add(None, "--must-staple") + arg_parser.add(None, "--validate-hooks") + + arg_parser.add(None, "-d", "--domain", dest="domains", + metavar="DOMAIN", action=_DomainsAction) + arg_parser.add(None, "--allow-subset-of-names") + # with self.assertRaises(errors.Error): + # arg_parser.parse_args() + + def test_parse_args_hosts_and_auto_hosts(self): + arg_parser = HelpfulArgumentParser(['--hsts', '--auto-hsts'], {}) + + arg_parser.add( + None, "--hsts", action="store_true", dest="hsts") + arg_parser.add( + None, "--auto-hsts", action="store_true", dest="auto_hsts") + # The following arguments are added because they have to be defined + # in order for arg_parser to run completely. They are not used for the + # test. + arg_parser.add( + None, constants.FORCE_INTERACTIVE_FLAG, action="store_true") + arg_parser.add( + None, "--non-interactive", dest="noninteractive_mode", + action="store_true") + arg_parser.add(None, "--staging") + arg_parser.add(None, "--dry-run") + arg_parser.add(None, "--csr") + arg_parser.add(None, "--must-staple") + arg_parser.add(None, "--validate-hooks") + arg_parser.add(None, "--allow-subset-of-names") + with self.assertRaises(errors.Error): + arg_parser.parse_args() + + +if __name__ == '__main__': + unittest.main() # pragma: no cover From 9819443440382695b74b77379d76e4886c0bdf70 Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Sat, 22 Feb 2020 15:22:27 +0100 Subject: [PATCH 17/92] Add test --- certbot-apache/tests/http_01_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 643a6bdd5..422a76443 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -1,5 +1,6 @@ """Test for certbot_apache._internal.http_01.""" import unittest +import errno import mock @@ -197,6 +198,12 @@ class ApacheHttp01Test(util.ApacheTest): self.assertTrue(os.path.exists(challenge_dir)) + @mock.patch("certbot_apache._internal.http_01.filesystem.makedirs") + def test_failed_makedirs(self, mock_makedirs): + mock_makedirs.side_effect = OSError(errno.EACCES, "msg") + self.http.add_chall(self.achalls[0]) + self.assertRaises(errors.PluginError, self.http.perform) + def _test_challenge_conf(self): with open(self.http.challenge_conf_pre) as f: pre_conf_contents = f.read() From d6ef34a03e21f8e8da0bc45d6b7124d50cfc5ec1 Mon Sep 17 00:00:00 2001 From: cumul Date: Tue, 19 Dec 2017 05:34:31 +0900 Subject: [PATCH 18/92] Use UTF-8 encoding for nginx plugin --- certbot-nginx/certbot_nginx/_internal/parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index f71d7c018..38c4d63e1 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -1,5 +1,6 @@ """NginxParser is a member object of the NginxConfigurator class.""" import copy +import codecs import functools import glob import logging @@ -205,7 +206,7 @@ class NginxParser(object): if item in self.parsed and not override: continue try: - with open(item) as _file: + with codecs.open(item, "r", "utf-8") as _file: parsed = nginxparser.load(_file) self.parsed[item] = parsed trees.append(parsed) @@ -414,7 +415,7 @@ class NginxParser(object): def _parse_ssl_options(ssl_options): if ssl_options is not None: try: - with open(ssl_options) as _file: + with codecs.open(ssl_options, "r", "utf-8") as _file: return nginxparser.load(_file) except IOError: logger.warning("Missing NGINX TLS options file: %s", ssl_options) From 247d9cd8870635dd9d065883e7853b86c0c76531 Mon Sep 17 00:00:00 2001 From: cumul Date: Wed, 31 Oct 2018 15:45:30 +0900 Subject: [PATCH 19/92] Use `io` module instead of `codecs` See https://mail.python.org/pipermail/python-list/2015-March/687124.html --- certbot-nginx/certbot_nginx/_internal/parser.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 38c4d63e1..d6182870c 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -1,8 +1,8 @@ """NginxParser is a member object of the NginxConfigurator class.""" import copy -import codecs import functools import glob +import io import logging import re @@ -206,12 +206,14 @@ class NginxParser(object): if item in self.parsed and not override: continue try: - with codecs.open(item, "r", "utf-8") as _file: + with io.open(item, "r", encoding="utf-8") as _file: parsed = nginxparser.load(_file) self.parsed[item] = parsed trees.append(parsed) except IOError: logger.warning("Could not open file: %s", item) + except UnicodeDecodeError: + logger.warning("Could not read file: %s due to invalid unicode character. Only UTF-8 encoding is supported.", item) except pyparsing.ParseException as err: logger.debug("Could not parse file: %s due to %s", item, err) return trees @@ -415,10 +417,12 @@ class NginxParser(object): def _parse_ssl_options(ssl_options): if ssl_options is not None: try: - with codecs.open(ssl_options, "r", "utf-8") as _file: + with io.open(ssl_options, "r", encoding="utf-8") as _file: return nginxparser.load(_file) except IOError: logger.warning("Missing NGINX TLS options file: %s", ssl_options) + except UnicodeDecodeError: + logger.warn("Could not read file: %s due to invalid unicode character. Only UTF-8 encoding is supported.", ssl_options) except pyparsing.ParseBaseException as err: logger.debug("Could not parse file: %s due to %s", ssl_options, err) return [] From 8b90b555186c0c51b0c37389375afcda2f27a3d6 Mon Sep 17 00:00:00 2001 From: cumul Date: Wed, 31 Oct 2018 16:48:01 +0900 Subject: [PATCH 20/92] Added test for valid/invalid unicode characters --- certbot-nginx/tests/parser_test.py | 12 ++++++++++++ .../testdata/etc_nginx/invalid_unicode_comments.conf | 7 +++++++ .../testdata/etc_nginx/valid_unicode_comments.conf | 9 +++++++++ 3 files changed, 28 insertions(+) create mode 100644 certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf create mode 100644 certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index f3a5665c5..232346396 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -482,6 +482,18 @@ class NginxParserTest(util.NginxTest): called = True self.assertTrue(called) + def test_valid_unicode_characters(self): + nparser = parser.NginxParser(self.config_path) + # pylint: disable=protected-access + parsed = nparser._parse_files(nparser.abs_path('unicode_support/valid_unicode_comments.conf')) + self.assertEqual(['server'], parsed[0][2][0]) + self.assertEqual(['listen', '80'], parsed[0][2][1][3]) + + def test_invalid_unicode_characters(self): + nparser = parser.NginxParser(self.config_path) + # pylint: disable=protected-access + parsed = nparser._parse_files(nparser.abs_path('unicode_support/invalid_unicode_comments.conf')) + self.assertEqual([], parsed) if __name__ == "__main__": diff --git a/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf b/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf new file mode 100644 index 000000000..596044cc9 --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf @@ -0,0 +1,7 @@ +# This configuration file is saved with EUC-KR (a.k.a. cp949) encoding, +# including some Korean alphabets. + +server { + # ȳϼ. 80 Ʈ û ٸ. + listen 80; +} diff --git a/certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf b/certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf new file mode 100644 index 000000000..89c978b2e --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf @@ -0,0 +1,9 @@ +# This configuration file is saved with valid UTF-8 encoding, +# including some CJK alphabets. + +server { + # 안녕하세요. 80번 포트에서 요청을 기다린다. + # こんにちは。８０番ポートからリクエストを待つ。 + # 你好。等待端口80上的请求。 + listen 80; +} From 0b21e716cabba503e4ec7ce7e722e93a319f361f Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 30 Jan 2019 15:12:55 -0800 Subject: [PATCH 21/92] Fix lint problems with long lines --- certbot-nginx/certbot_nginx/_internal/parser.py | 7 +++++-- certbot-nginx/tests/parser_test.py | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index d6182870c..72d4d38de 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -213,7 +213,9 @@ class NginxParser(object): except IOError: logger.warning("Could not open file: %s", item) except UnicodeDecodeError: - logger.warning("Could not read file: %s due to invalid unicode character. Only UTF-8 encoding is supported.", item) + logger.warning("Could not read file: %s due to invalid " + "character. Only UTF-8 encoding is " + "supported.", item) except pyparsing.ParseException as err: logger.debug("Could not parse file: %s due to %s", item, err) return trees @@ -422,7 +424,8 @@ def _parse_ssl_options(ssl_options): except IOError: logger.warning("Missing NGINX TLS options file: %s", ssl_options) except UnicodeDecodeError: - logger.warn("Could not read file: %s due to invalid unicode character. Only UTF-8 encoding is supported.", ssl_options) + logger.warn("Could not read file: %s due to invalid character. " + "Only UTF-8 encoding is supported.", ssl_options) except pyparsing.ParseBaseException as err: logger.debug("Could not parse file: %s due to %s", ssl_options, err) return [] diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 232346396..632ea179f 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -485,14 +485,16 @@ class NginxParserTest(util.NginxTest): def test_valid_unicode_characters(self): nparser = parser.NginxParser(self.config_path) # pylint: disable=protected-access - parsed = nparser._parse_files(nparser.abs_path('unicode_support/valid_unicode_comments.conf')) + path = nparser.abs_path('unicode_support/valid_unicode_comments.conf') + parsed = nparser._parse_files(path) self.assertEqual(['server'], parsed[0][2][0]) self.assertEqual(['listen', '80'], parsed[0][2][1][3]) def test_invalid_unicode_characters(self): nparser = parser.NginxParser(self.config_path) # pylint: disable=protected-access - parsed = nparser._parse_files(nparser.abs_path('unicode_support/invalid_unicode_comments.conf')) + path = nparser.abs_path('unicode_support/invalid_unicode_comments.conf') + parsed = nparser._parse_files(path) self.assertEqual([], parsed) From c3cfd412c91cfdc2d0ba95ad4c3751324bf8b132 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 01:46:41 +0900 Subject: [PATCH 22/92] Relpace deprecated `logger.warn()` with `logger.warning()` --- certbot-nginx/certbot_nginx/_internal/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 72d4d38de..0c1151826 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -424,8 +424,8 @@ def _parse_ssl_options(ssl_options): except IOError: logger.warning("Missing NGINX TLS options file: %s", ssl_options) except UnicodeDecodeError: - logger.warn("Could not read file: %s due to invalid character. " - "Only UTF-8 encoding is supported.", ssl_options) + logger.warning("Could not read file: %s due to invalid character. " + "Only UTF-8 encoding is supported.", ssl_options) except pyparsing.ParseBaseException as err: logger.debug("Could not parse file: %s due to %s", ssl_options, err) return [] From 22685ef86fa2f00be647403f3c64cec79b4d2be1 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:23:46 +0900 Subject: [PATCH 23/92] Remove `unicode_support/` path in test case --- certbot-nginx/tests/parser_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 632ea179f..d766a6a9b 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -485,7 +485,7 @@ class NginxParserTest(util.NginxTest): def test_valid_unicode_characters(self): nparser = parser.NginxParser(self.config_path) # pylint: disable=protected-access - path = nparser.abs_path('unicode_support/valid_unicode_comments.conf') + path = nparser.abs_path('valid_unicode_comments.conf') parsed = nparser._parse_files(path) self.assertEqual(['server'], parsed[0][2][0]) self.assertEqual(['listen', '80'], parsed[0][2][1][3]) @@ -493,7 +493,7 @@ class NginxParserTest(util.NginxTest): def test_invalid_unicode_characters(self): nparser = parser.NginxParser(self.config_path) # pylint: disable=protected-access - path = nparser.abs_path('unicode_support/invalid_unicode_comments.conf') + path = nparser.abs_path('invalid_unicode_comments.conf') parsed = nparser._parse_files(path) self.assertEqual([], parsed) From 36311a276b7ffbff9bcc8b019f6a12f97e153712 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:46:27 +0900 Subject: [PATCH 24/92] Add test case for `_parse_ssl_options()` --- certbot-nginx/tests/parser_test.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index d766a6a9b..a0484f965 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -497,6 +497,25 @@ class NginxParserTest(util.NginxTest): parsed = nparser._parse_files(path) self.assertEqual([], parsed) + def test_valid_unicode_characters_in_ssl_options(self): + nparser = parser.NginxParser(self.config_path) + path = nparser.abs_path('valid_unicode_comments.conf') + parsed = parser._parse_ssl_options(path) # pylint: disable=protected-access + self.assertEqual(['server'], parsed[2][0]) + self.assertEqual(['listen', '80'], parsed[2][1][3]) + + def test_invalid_unicode_characters_in_ssl_options(self): + with self.assertLogs() as log: + nparser = parser.NginxParser(self.config_path) + path = nparser.abs_path('invalid_unicode_comments.conf') + parsed = parser._parse_ssl_options(path) # pylint: disable=protected-access + + self.assertEqual([], parsed) + self.assertTrue([ + True + for output in log.output + if ('invalid character' in output) and ('UTF-8' in output) + ]) if __name__ == "__main__": unittest.main() # pragma: no cover From 20df5507ae9a17df35f9c1152a34d0c52ba304f6 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:47:44 +0900 Subject: [PATCH 25/92] Add logging test for `_parse_files()` --- certbot-nginx/tests/parser_test.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index a0484f965..89bcf689a 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -491,11 +491,17 @@ class NginxParserTest(util.NginxTest): self.assertEqual(['listen', '80'], parsed[0][2][1][3]) def test_invalid_unicode_characters(self): - nparser = parser.NginxParser(self.config_path) - # pylint: disable=protected-access - path = nparser.abs_path('invalid_unicode_comments.conf') - parsed = nparser._parse_files(path) + with self.assertLogs() as log: + nparser = parser.NginxParser(self.config_path) + path = nparser.abs_path('invalid_unicode_comments.conf') + parsed = nparser._parse_files(path) # pylint: disable=protected-access + self.assertEqual([], parsed) + self.assertTrue([ + True + for output in log.output + if ('invalid character' in output) and ('UTF-8' in output) + ]) def test_valid_unicode_characters_in_ssl_options(self): nparser = parser.NginxParser(self.config_path) From 2aac24c9829d565d6ba6aa42ad9fc7e5ba4926b3 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:48:21 +0900 Subject: [PATCH 26/92] Trivial code clean-up --- certbot-nginx/tests/parser_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 89bcf689a..1f9e3c996 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -484,9 +484,8 @@ class NginxParserTest(util.NginxTest): def test_valid_unicode_characters(self): nparser = parser.NginxParser(self.config_path) - # pylint: disable=protected-access path = nparser.abs_path('valid_unicode_comments.conf') - parsed = nparser._parse_files(path) + parsed = nparser._parse_files(path) # pylint: disable=protected-access self.assertEqual(['server'], parsed[0][2][0]) self.assertEqual(['listen', '80'], parsed[0][2][1][3]) From b3071aab29c480d52837e70fcf3fed9379f22200 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:52:41 +0900 Subject: [PATCH 27/92] Add my name to AUTHORS.md :) --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 80a24d3be..8653382b8 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -268,3 +268,4 @@ Authors * [YourDaddyIsHere](https://github.com/YourDaddyIsHere) * [Zach Shepherd](https://github.com/zjs) * [陈三](https://github.com/chenxsan) +* [Yuseong Cho](https://github.com/g6123) From d68f37ae88d32dcca97785df9ec4c3cf70c437ae Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 02:56:43 +0900 Subject: [PATCH 28/92] Add this change to CHANGELOG.md --- certbot/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 126b07eec..837f48db2 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -17,7 +17,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* +* Fix nginx plguin crash when non-ASCII configuration file is being read More details about these changes can be found on our GitHub repo. From 32904d8c9e2c1c6dd253eb860472053053238238 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 04:26:37 +0900 Subject: [PATCH 29/92] Add `TestCase.assertLogs()` backport for Python 2.7 --- certbot-nginx/tests/test_log_util.py | 123 +++++++++++++++++++++++++++ certbot-nginx/tests/test_util.py | 3 +- 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 certbot-nginx/tests/test_log_util.py diff --git a/certbot-nginx/tests/test_log_util.py b/certbot-nginx/tests/test_log_util.py new file mode 100644 index 000000000..f8135ca26 --- /dev/null +++ b/certbot-nginx/tests/test_log_util.py @@ -0,0 +1,123 @@ +"""Backport for `TestCase.assertLogs()`. + +Most of the idea and code are from CPython implementation. +https://github.com/python/cpython/blob/b76518d43fb82ed9e5d27025d18c90a23d525c90/Lib/unittest/case.py +""" +import logging +import collections + +__all__ = ['AssertLogsMixin'] + +LoggingWatcher = collections.namedtuple('LoggingWatcher', ['records', 'output']) + + +class CapturingHandler(logging.Handler): + """ + A logging handler capturing all (raw and formatted) logging output. + """ + + def __init__(self): + super(CapturingHandler, self).__init__() + self.watcher = LoggingWatcher([], []) + + def flush(self): + pass + + def emit(self, record): + self.watcher.records.append(record) + self.watcher.output.append(self.format(record)) + + + +class AssertLogsContext(object): + """ + A context manager used to implement `TestCase.assertLogs()`. + """ + + LOGGING_FORMAT = '%(levelname)s:%(name)s:%(message)s' + + def __init__(self, test_case, logger_name, level): + self.test_case = test_case + + self.logger_name = logger_name + self.logger_states = None + self.logger = None + + if level: + # pylint: disable=protected-access,no-member + try: + name_to_level = logging._nameToLevel # type: ignore + except AttributeError: + name_to_level = logging._levelNames # type: ignore + + self.level = name_to_level.get(level, level) + else: + self.level = logging.INFO + + self.watcher = None + + def _save_logger_states(self): + self.logger_states = (self.logger.handlers[:], self.logger.level, self.logger.propagate) + + def _restore_logger_states(self): + self.logger.handlers, self.logger.level, self.logger.propagate = self.logger_states + + def __enter__(self): + if isinstance(self.logger_name, logging.Logger): + self.logger = self.logger_name + else: + self.logger = logging.getLogger(self.logger_name) + + formatter = logging.Formatter(self.LOGGING_FORMAT) + + handler = CapturingHandler() + handler.setFormatter(formatter) + + self._save_logger_states() + self.logger.handlers = [handler] + self.logger.setLevel(self.level) + self.logger.propagate = False + + self.watcher = handler.watcher + return handler.watcher + + def __exit__(self, exc_type, exc_value, tb): + self._restore_logger_states() + + if exc_type is not None: + # let unexpected exceptions pass through + return + + if not self.watcher.records: + self._raiseFailure( + "no logs of level {} or higher triggered on {}" + .format(logging.getLevelName(self.level), self.logger.name)) + + def _raiseFailure(self, message): + message = self.test_case._formatMessage(None, message) # pylint: disable=protected-access + raise self.test_case.failureException(message) + + +class AssertLogsMixin(object): + """ + A mixin that implements `TestCase.assertLogs()`. + """ + + def assertLogs(self, logger=None, level=None): + """Fail unless a log message of level *level* or higher is emitted + on *logger_name* or its children. If omitted, *level* defaults to + INFO and *logger* defaults to the root logger. + This method must be used as a context manager, and will yield + a recording object with two attributes: `output` and `records`. + At the end of the context manager, the `output` attribute will + be a list of the matching formatted log messages and the + `records` attribute will be a list of the corresponding LogRecord + objects. + Example:: + with self.assertLogs('foo', level='INFO') as cm: + logging.getLogger('foo').info('first message') + logging.getLogger('foo.bar').error('second message') + self.assertEqual(cm.output, ['INFO:foo:first message', + 'ERROR:foo.bar:second message']) + """ + return AssertLogsContext(self, logger, level) diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index 8dfd18637..4c9da84bd 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -14,9 +14,10 @@ from certbot.plugins import common from certbot.tests import util as test_util from certbot_nginx._internal import configurator from certbot_nginx._internal import nginxparser +import test_log_util -class NginxTest(test_util.ConfigTestCase): +class NginxTest(test_log_util.AssertLogsMixin, test_util.ConfigTestCase): def setUp(self): super(NginxTest, self).setUp() From 5b29e4616c4a7bae3ba18b0eca8ae245afbf97f1 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Mon, 24 Feb 2020 05:30:54 +0900 Subject: [PATCH 30/92] Add simple comments --- certbot-nginx/tests/test_log_util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot-nginx/tests/test_log_util.py b/certbot-nginx/tests/test_log_util.py index f8135ca26..7aebf2151 100644 --- a/certbot-nginx/tests/test_log_util.py +++ b/certbot-nginx/tests/test_log_util.py @@ -46,8 +46,10 @@ class AssertLogsContext(object): if level: # pylint: disable=protected-access,no-member try: + # In Python 3.x name_to_level = logging._nameToLevel # type: ignore except AttributeError: + # In Python 2.7 name_to_level = logging._levelNames # type: ignore self.level = name_to_level.get(level, level) From 2633c3ffb6a4f66933daef238b6a140ffc059818 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Mon, 24 Feb 2020 07:49:42 +1100 Subject: [PATCH 31/92] acme: ignore params in content-type check (#7342) * acme: ignore params in content-type check Fixes the warning in #7339 * Suppress coverage complaint in test * Update CHANGELOG * Repair symlink Co-authored-by: Adrien Ferrand --- acme/acme/client.py | 3 +++ acme/tests/client_test.py | 29 +++++++++++++++++++++++++++++ certbot/CHANGELOG.md | 1 + 3 files changed, 33 insertions(+) diff --git a/acme/acme/client.py b/acme/acme/client.py index 3e03748b5..cecb727c7 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1022,6 +1022,9 @@ class ClientNetwork(object): """ response_ct = response.headers.get('Content-Type') + # Strip parameters from the media-type (rfc2616#section-3.7) + if response_ct: + response_ct = response_ct.split(';')[0].strip() try: # TODO: response.json() is called twice, once here, and # once in _get and _post clients diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index a38fedbd6..a4966140f 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -980,6 +980,35 @@ class ClientNetworkTest(unittest.TestCase): self.assertEqual( self.response, self.net._check_response(self.response)) + @mock.patch('acme.client.logger') + def test_check_response_ok_ct_with_charset(self, mock_logger): + self.response.json.return_value = {} + self.response.headers['Content-Type'] = 'application/json; charset=utf-8' + # pylint: disable=protected-access + self.assertEqual(self.response, self.net._check_response( + self.response, content_type='application/json')) + try: + mock_logger.debug.assert_called_with( + 'Ignoring wrong Content-Type (%r) for JSON decodable response', + 'application/json; charset=utf-8' + ) + except AssertionError: + return + raise AssertionError('Expected Content-Type warning ' #pragma: no cover + 'to not have been logged') + + @mock.patch('acme.client.logger') + def test_check_response_ok_bad_ct(self, mock_logger): + self.response.json.return_value = {} + self.response.headers['Content-Type'] = 'text/plain' + # pylint: disable=protected-access + self.assertEqual(self.response, self.net._check_response( + self.response, content_type='application/json')) + mock_logger.debug.assert_called_with( + 'Ignoring wrong Content-Type (%r) for JSON decodable response', + 'text/plain' + ) + def test_check_response_conflict(self): self.response.ok = False self.response.status_code = 409 diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 126b07eec..bc5ad90d6 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -14,6 +14,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * certbot._internal.cli is now a package split in submodules instead of a whole module. +* Fix acme module warnings when response Content-Type includes params (e.g. charset). ### Fixed From 4fd04366aad02e2fa51057d4912346bde9e39d02 Mon Sep 17 00:00:00 2001 From: martin-c Date: Sun, 23 Feb 2020 22:14:51 +0100 Subject: [PATCH 32/92] Fix issue #7165 in _create_challenge_dirs(), attempt to fix pylint errors (#7568) * fix issue #7165 by checking if directory exists before trying to create it, fix possible pylint issues in webroot.py * fix get_chall_pref definition * Update CHANGELOG.md * Update CHANGELOG.md Co-authored-by: Adrien Ferrand --- certbot/CHANGELOG.md | 2 ++ certbot/certbot/_internal/plugins/webroot.py | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index bc5ad90d6..30479f25b 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -15,6 +15,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * certbot._internal.cli is now a package split in submodules instead of a whole module. * Fix acme module warnings when response Content-Type includes params (e.g. charset). +* Fixed issue where webroot plugin would incorrectly raise `Read-only file system` + error when creating challenge directories (issue #7165). ### Fixed diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index 042b60656..9383ce66d 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -1,7 +1,6 @@ """Webroot plugin.""" import argparse import collections -import errno import json import logging @@ -71,7 +70,7 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} # type: Dict[str, str] self.performed = collections.defaultdict(set) \ - # type: DefaultDict[str, Set[achallenges.KeyAuthorizationAnnotatedChallenge]] + # type: DefaultDict[str, Set[achallenges.KeyAuthorizationAnnotatedChallenge]] # stack of dirs successfully created by this authenticator self._created_dirs = [] # type: List[str] @@ -137,7 +136,7 @@ to serve all files under specified web root ({0}).""" "webroot when using the webroot plugin.") return None if index == 0 else known_webroots[index - 1] # code == display_util.OK - def _prompt_for_new_webroot(self, domain, allowraise=False): + def _prompt_for_new_webroot(self, domain, allowraise=False): # pylint: no-self-use code, webroot = ops.validated_directory( _validate_webroot, "Input the webroot for {0}:".format(domain), @@ -170,6 +169,10 @@ to serve all files under specified web root ({0}).""" # We ignore the last prefix in the next iteration, # as it does not correspond to a folder path ('/' or 'C:') for prefix in sorted(util.get_prefixes(self.full_roots[name])[:-1], key=len): + if os.path.isdir(prefix): + # Don't try to create directory if it already exists, as some filesystems + # won't reliably raise EEXIST or EISDIR if directory exists. + continue try: # Set owner as parent directory if possible, apply mode for Linux/Windows. # For Linux, this is coupled with the "umask" call above because @@ -184,14 +187,13 @@ to serve all files under specified web root ({0}).""" logger.info("Unable to change owner and uid of webroot directory") logger.debug("Error was: %s", exception) except OSError as exception: - if exception.errno not in (errno.EEXIST, errno.EISDIR): - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}".format(name, exception)) + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}".format(name, exception)) finally: os.umask(old_umask) - def _get_validation_path(self, root_path, achall): + def _get_validation_path(self, root_path, achall): # pylint: no-self-use return os.path.join(root_path, achall.chall.encode("token")) def _perform_single(self, achall): From 4ea98d830bcc3d1b980a4055243c6a6a25d8dc54 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 24 Feb 2020 12:31:16 -0800 Subject: [PATCH 33/92] remove _internal docs (#7801) --- certbot/CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 30479f25b..57fbc820b 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -13,7 +13,6 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* certbot._internal.cli is now a package split in submodules instead of a whole module. * Fix acme module warnings when response Content-Type includes params (e.g. charset). * Fixed issue where webroot plugin would incorrectly raise `Read-only file system` error when creating challenge directories (issue #7165). From 2ae090529e66a059e0e077d233f427583caa7c6c Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Tue, 25 Feb 2020 13:17:25 +0900 Subject: [PATCH 34/92] Fixed typo & some trivial documentation change --- AUTHORS.md | 2 +- certbot/CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index 8653382b8..f5b981b8e 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -266,6 +266,6 @@ Authors * [Yomna](https://github.com/ynasser) * [Yoni Jah](https://github.com/yonjah) * [YourDaddyIsHere](https://github.com/YourDaddyIsHere) +* [Yuseong Cho](https://github.com/g6123) * [Zach Shepherd](https://github.com/zjs) * [陈三](https://github.com/chenxsan) -* [Yuseong Cho](https://github.com/g6123) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 837f48db2..a68c50ad0 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -17,7 +17,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* Fix nginx plguin crash when non-ASCII configuration file is being read +* Fix nginx plugin crash when non-ASCII configuration file is being read More details about these changes can be found on our GitHub repo. From ddf68aea8039b81940b7caf731405ef10613b5db Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Tue, 25 Feb 2020 13:21:10 +0900 Subject: [PATCH 35/92] Update comment in testdata file --- .../tests/testdata/etc_nginx/invalid_unicode_comments.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf b/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf index 596044cc9..4d6384535 100644 --- a/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf +++ b/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf @@ -1,5 +1,5 @@ # This configuration file is saved with EUC-KR (a.k.a. cp949) encoding, -# including some Korean alphabets. +# including some Korean letters. server { # ȳϼ. 80 Ʈ û ٸ. From a48907920885671ef5a57bc1cda556e90c5e0e20 Mon Sep 17 00:00:00 2001 From: cumul0529 Date: Tue, 25 Feb 2020 13:26:36 +0900 Subject: [PATCH 36/92] Update parser test to better assert logging output --- certbot-nginx/tests/parser_test.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 1f9e3c996..fd5d338d3 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -496,11 +496,10 @@ class NginxParserTest(util.NginxTest): parsed = nparser._parse_files(path) # pylint: disable=protected-access self.assertEqual([], parsed) - self.assertTrue([ - True + self.assertTrue(any( + ('invalid character' in output) and ('UTF-8' in output) for output in log.output - if ('invalid character' in output) and ('UTF-8' in output) - ]) + )) def test_valid_unicode_characters_in_ssl_options(self): nparser = parser.NginxParser(self.config_path) @@ -516,11 +515,10 @@ class NginxParserTest(util.NginxTest): parsed = parser._parse_ssl_options(path) # pylint: disable=protected-access self.assertEqual([], parsed) - self.assertTrue([ - True + self.assertTrue(any( + ('invalid character' in output) and ('UTF-8' in output) for output in log.output - if ('invalid character' in output) and ('UTF-8' in output) - ]) + )) if __name__ == "__main__": unittest.main() # pragma: no cover From f4c0a9fd63c9be4cd4e745dd5f701040bcd14682 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 10:43:41 -0800 Subject: [PATCH 37/92] Split advanced pipeline (#7813) I want to do what I did in https://github.com/certbot/certbot/pull/7733 to our Azure Pipelines setup, but unfortunately this isn't currently possible. The only filters available for service hooks for the "build completed" trigger are the pipeline and build status. See ![Screen Shot 2020-02-26 at 3 04 56 PM](https://user-images.githubusercontent.com/6504915/75396464-64ad0780-58a9-11ea-97a1-3454a9754675.png) To accomplish this, I propose splitting the "advanced" pipeline into two cases. One is for builds on protected branches where we want to be notified if they fail while the other is just used to manually run tests on certain branches. --- .azure-pipelines/advanced-test.yml | 12 ++++++++++++ .azure-pipelines/advanced.yml | 10 ++-------- .azure-pipelines/release.yml | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 .azure-pipelines/advanced-test.yml diff --git a/.azure-pipelines/advanced-test.yml b/.azure-pipelines/advanced-test.yml new file mode 100644 index 000000000..b9ac9c38a --- /dev/null +++ b/.azure-pipelines/advanced-test.yml @@ -0,0 +1,12 @@ +# Advanced pipeline for running our full test suite on demand. +trigger: + # When changing these triggers, please ensure the documentation under + # "Running tests in CI" is still correct. + - azure-test-* + - test-* + +jobs: + # Any addition here should be reflected in the advanced and release pipelines. + # It is advised to declare all jobs here as templates to improve maintainability. + - template: templates/tests-suite.yml + - template: templates/installer-tests.yml diff --git a/.azure-pipelines/advanced.yml b/.azure-pipelines/advanced.yml index dda7f9bfd..7f0f5de50 100644 --- a/.azure-pipelines/advanced.yml +++ b/.azure-pipelines/advanced.yml @@ -1,12 +1,6 @@ -# Advanced pipeline for isolated checks and release purpose +# Advanced pipeline for running our full test suite on protected branches. trigger: - # When changing these triggers, please ensure the documentation under - # "Running tests in CI" is still correct. - - azure-test-* - - test-* - '*.x' -pr: - - test-* # This pipeline is also nightly run on master schedules: - cron: "0 4 * * *" @@ -17,7 +11,7 @@ schedules: always: true jobs: - # Any addition here should be reflected in the release pipeline. + # Any addition here should be reflected in the advanced-test and release pipelines. # It is advised to declare all jobs here as templates to improve maintainability. - template: templates/tests-suite.yml - template: templates/installer-tests.yml diff --git a/.azure-pipelines/release.yml b/.azure-pipelines/release.yml index aeb5ee327..e9acbc69a 100644 --- a/.azure-pipelines/release.yml +++ b/.azure-pipelines/release.yml @@ -6,7 +6,7 @@ trigger: pr: none jobs: - # Any addition here should be reflected in the advanced pipeline. + # Any addition here should be reflected in the advanced and advanced-test pipelines. # It is advised to declare all jobs here as templates to improve maintainability. - template: templates/tests-suite.yml - template: templates/installer-tests.yml From 24aa1e9127802f9c6ac459bbf91e6ff9b4595483 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 10:47:43 -0800 Subject: [PATCH 38/92] update letstest reqs (#7809) I don't fully understand why, but since I updated my macbook to macOS Catalina, the test script currently fails to run for me with the versions of our dependencies we have pinned. Updating the dependencies solves the problem though and you can see Travis also successfully running tests with these new dependencies at https://travis-ci.com/certbot/certbot/builds/150573696. --- tests/letstest/requirements.txt | 38 ++++++++++++++------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/tests/letstest/requirements.txt b/tests/letstest/requirements.txt index 64e1f6a0c..24bd77331 100644 --- a/tests/letstest/requirements.txt +++ b/tests/letstest/requirements.txt @@ -1,25 +1,19 @@ -asn1crypto==0.24.0 -awscli==1.16.157 -bcrypt==3.1.6 -boto3==1.9.146 -botocore==1.12.147 -cffi==1.12.3 -colorama==0.3.9 -cryptography==2.4.2 -docutils==0.14 -enum34==1.1.6 +bcrypt==3.1.7 +boto3==1.12.7 +botocore==1.15.7 +cffi==1.14.0 +cryptography==2.8 +docutils==0.15.2 +enum34==1.1.9 Fabric==1.14.1 -futures==3.2.0 -idna==2.8 -ipaddress==1.0.22 -jmespath==0.9.4 -paramiko==2.4.2 -pyasn1==0.4.5 +futures==3.3.0 +ipaddress==1.0.23 +jmespath==0.9.5 +paramiko==2.7.1 pycparser==2.19 PyNaCl==1.3.0 -python-dateutil==2.8.0 -PyYAML==3.10 -rsa==3.4.2 -s3transfer==0.2.0 -six==1.12.0 -urllib3==1.24.3 +python-dateutil==2.8.1 +PyYAML==5.3 +s3transfer==0.3.3 +six==1.14.0 +urllib3==1.25.8 From 8c75a9de9fe28ca0e4baf8686620a9c6b5733515 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 10:47:56 -0800 Subject: [PATCH 39/92] Remove unused notify code. (#7805) This code is unused and hasn't been modified since 2015 except for various times our files have been renamed. Let's remove it. --- certbot/certbot/_internal/notify.py | 34 ------------------- certbot/tests/notify_test.py | 52 ----------------------------- 2 files changed, 86 deletions(-) delete mode 100644 certbot/certbot/_internal/notify.py delete mode 100644 certbot/tests/notify_test.py diff --git a/certbot/certbot/_internal/notify.py b/certbot/certbot/_internal/notify.py deleted file mode 100644 index dda0a85af..000000000 --- a/certbot/certbot/_internal/notify.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Send e-mail notification to system administrators.""" - -import email -import smtplib -import socket -import subprocess - - -def notify(subject, whom, what): - """Send email notification. - - Try to notify the addressee (``whom``) by e-mail, with Subject: - defined by ``subject`` and message body by ``what``. - - """ - msg = email.message_from_string(what) - msg.add_header("From", "Certbot renewal agent ") - msg.add_header("To", whom) - msg.add_header("Subject", subject) - msg = msg.as_string() - try: - lmtp = smtplib.LMTP() - lmtp.connect() - lmtp.sendmail("root", [whom], msg) - except (smtplib.SMTPHeloError, smtplib.SMTPRecipientsRefused, - smtplib.SMTPSenderRefused, smtplib.SMTPDataError, socket.error): - # We should try using /usr/sbin/sendmail in this case - try: - proc = subprocess.Popen(["/usr/sbin/sendmail", "-t"], - stdin=subprocess.PIPE) - proc.communicate(msg) - except OSError: - return False - return True diff --git a/certbot/tests/notify_test.py b/certbot/tests/notify_test.py deleted file mode 100644 index d6f7d2239..000000000 --- a/certbot/tests/notify_test.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Tests for certbot._internal.notify.""" -import socket -import unittest - -import mock - - -class NotifyTests(unittest.TestCase): - """Tests for the notifier.""" - - @mock.patch("certbot._internal.notify.smtplib.LMTP") - def test_smtp_success(self, mock_lmtp): - from certbot._internal.notify import notify - lmtp_obj = mock.MagicMock() - mock_lmtp.return_value = lmtp_obj - self.assertTrue(notify("Goose", "auntrhody@example.com", - "The old grey goose is dead.")) - self.assertEqual(lmtp_obj.connect.call_count, 1) - self.assertEqual(lmtp_obj.sendmail.call_count, 1) - - @mock.patch("certbot._internal.notify.smtplib.LMTP") - @mock.patch("certbot._internal.notify.subprocess.Popen") - def test_smtp_failure(self, mock_popen, mock_lmtp): - from certbot._internal.notify import notify - lmtp_obj = mock.MagicMock() - mock_lmtp.return_value = lmtp_obj - lmtp_obj.sendmail.side_effect = socket.error(17) - proc = mock.MagicMock() - mock_popen.return_value = proc - self.assertTrue(notify("Goose", "auntrhody@example.com", - "The old grey goose is dead.")) - self.assertEqual(lmtp_obj.sendmail.call_count, 1) - self.assertEqual(proc.communicate.call_count, 1) - - @mock.patch("certbot._internal.notify.smtplib.LMTP") - @mock.patch("certbot._internal.notify.subprocess.Popen") - def test_everything_fails(self, mock_popen, mock_lmtp): - from certbot._internal.notify import notify - lmtp_obj = mock.MagicMock() - mock_lmtp.return_value = lmtp_obj - lmtp_obj.sendmail.side_effect = socket.error(17) - proc = mock.MagicMock() - mock_popen.return_value = proc - proc.communicate.side_effect = OSError("What we have here is a " - "failure to communicate.") - self.assertFalse(notify("Goose", "auntrhody@example.com", - "The old grey goose is dead.")) - self.assertEqual(lmtp_obj.sendmail.call_count, 1) - self.assertEqual(proc.communicate.call_count, 1) - -if __name__ == "__main__": - unittest.main() # pragma: no cover From 2f737ee292680e2f8043e0dfe3affcccc03914e8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 10:49:50 -0800 Subject: [PATCH 40/92] Change how _USE_DISTRO is set for mypy (#7804) If you run `mypy --platform darwin certbot/certbot/util.py` you'll get: ``` certbot/certbot/util.py:303: error: Name 'distro' is not defined certbot/certbot/util.py:319: error: Name 'distro' is not defined certbot/certbot/util.py:369: error: Name 'distro' is not defined ``` This is because mypy's logic for handling platform specific code is pretty simple and can't figure out what we're doing with `_USE_DISTRO` here. See https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks for more info. Setting `_USE_DISTRO` to the result of `sys.platform.startswith('linux')` solves the problem without changing the overall behavior of our code here though. This fixes part of https://github.com/certbot/certbot/issues/7803, but there's more work to be done on Windows. --- certbot/certbot/util.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index aff2952f7..e69b11543 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -25,11 +25,9 @@ from certbot._internal import lock from certbot.compat import filesystem from certbot.compat import os -if sys.platform.startswith('linux'): +_USE_DISTRO = sys.platform.startswith('linux') +if _USE_DISTRO: import distro - _USE_DISTRO = True -else: - _USE_DISTRO = False logger = logging.getLogger(__name__) From a2be8e1956c79662fd28d8b8af4802ea89cf29bf Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 10:50:20 -0800 Subject: [PATCH 41/92] Fix tests on macOS Catalina (#7794) This PR fixes the failures that can be seen at https://dev.azure.com/certbot/certbot/_build/results?buildId=1184&view=results. You can see this code running on macOS Catalina at https://dev.azure.com/certbot/certbot/_build/results?buildId=1192&view=results. --- certbot/tests/cli_test.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 3a7fb57f8..be2c8f29e 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -30,15 +30,23 @@ class TestReadFile(TempDirTestCase): # However a relative path between two different drives is invalid. So we move to # self.tempdir to ensure that we stay on the same drive. os.chdir(self.tempdir) - rel_test_path = os.path.relpath(os.path.join(self.tempdir, 'foo')) + # The read-only filesystem introduced with macOS Catalina can break + # code using relative paths below. See + # https://bugs.python.org/issue38295 for another example of this. + # Eliminating any possible symlinks in self.tempdir before passing + # it to os.path.relpath solves the problem. This is done by calling + # filesystem.realpath which removes any symlinks in the path on + # POSIX systems. + real_path = filesystem.realpath(os.path.join(self.tempdir, 'foo')) + relative_path = os.path.relpath(real_path) self.assertRaises( - argparse.ArgumentTypeError, cli.read_file, rel_test_path) + argparse.ArgumentTypeError, cli.read_file, relative_path) test_contents = b'bar\n' - with open(rel_test_path, 'wb') as f: + with open(relative_path, 'wb') as f: f.write(test_contents) - path, contents = cli.read_file(rel_test_path) + path, contents = cli.read_file(relative_path) self.assertEqual(path, os.path.abspath(path)) self.assertEqual(contents, test_contents) finally: From 6309ded92f03104f2baa9b881db9827f5fe11e4c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 14:43:28 -0800 Subject: [PATCH 42/92] Remove references to deprecated flags in Certbot. (#7509) Related to https://github.com/certbot/certbot/pull/7482, this removes some references to deprecated options in Certbot. The only references I didn't remove were: * In `certbot/tests/testdata/sample-renewal*` which contains a lot of old values and I think there's even some value in keeping them so we know if we make a change that suddenly causes old renewal configuration files to error. * In the Apache and Nginx plugins and I created https://github.com/certbot/certbot/issues/7508 to resolve that issue. --- certbot/certbot/_internal/main.py | 2 +- certbot/certbot/display/ops.py | 2 +- certbot/tests/cli_test.py | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 8674cd151..4a57dd78d 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -394,7 +394,7 @@ def _find_domains_or_certname(config, installer, question=None): :param installer: Installer object :type installer: interfaces.IInstaller - :param `str` question: Overriding dialog question to ask the user if asked + :param `str` question: Overriding default question to ask the user if asked to choose from domain names. :returns: Two-part tuple of domains and certname diff --git a/certbot/certbot/display/ops.py b/certbot/certbot/display/ops.py index eab9d251d..f24f6ed99 100644 --- a/certbot/certbot/display/ops.py +++ b/certbot/certbot/display/ops.py @@ -107,7 +107,7 @@ def choose_names(installer, question=None): :param installer: An installer object :type installer: :class:`certbot.interfaces.IInstaller` - :param `str` question: Overriding dialog question to ask the user if asked + :param `str` question: Overriding default question to ask the user if asked to choose from domain names. :returns: List of selected names diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index be2c8f29e..7d21f8bb8 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -150,7 +150,6 @@ class ParseTest(unittest.TestCase): self.assertTrue("how a certificate is deployed" in out) self.assertTrue("--webroot-path" in out) self.assertTrue("--text" not in out) - self.assertTrue("--dialog" not in out) self.assertTrue("%s" not in out) self.assertTrue("{0}" not in out) self.assertTrue("--renew-hook" not in out) @@ -211,7 +210,6 @@ class ParseTest(unittest.TestCase): self.assertTrue("how a certificate is deployed" in out) self.assertTrue("--webroot-path" in out) self.assertTrue("--text" not in out) - self.assertTrue("--dialog" not in out) self.assertTrue("%s" not in out) self.assertTrue("{0}" not in out) From fa67b7ba0fb03453fc8d03e3631d6782a54a233b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 14:44:39 -0800 Subject: [PATCH 43/92] Remove codecov (#7811) After getting a +1 from everyone on the team, this PR removes the use of `codecov` from the Certbot repo because we keep having problems with it. Two noteworthy things about this PR are: 1. I left the text at https://github.com/certbot/certbot/blob/4ea98d830bcc3d1b980a4055243c6a6a25d8dc54/.azure-pipelines/INSTALL.md#add-a-secret-variable-to-a-pipeline-like-codecov_token because I think it's useful to document how to set up a secret variable in general. 2. I'm not sure what the text "Option -e makes sure we fail fast and don't submit to codecov." in `tox.cover.py` refers to but it seems incorrect since `-e` isn't accepted or used by the script so I just deleted the line. As part of this, I said I'd open an issue to track setting up coveralls (which seems to be the only real alternative to codecov) which is at https://github.com/certbot/certbot/issues/7810. With my change, failure output looks something like: ``` $ tox -e py27-cover ... Name Stmts Miss Cover Missing ------------------------------------------------------------------------------------------ certbot/certbot/__init__.py 1 0 100% certbot/certbot/_internal/__init__.py 0 0 100% certbot/certbot/_internal/account.py 191 4 98% 62-63, 206, 337 ... certbot/tests/storage_test.py 530 0 100% certbot/tests/util_test.py 374 29 92% 211-213, 480-484, 489-499, 504-511, 545-547, 552-554 ------------------------------------------------------------------------------------------ TOTAL 14451 647 96% Command '['/path/to/certbot/dir/.tox/py27-cover/bin/python', '-m', 'coverage', 'report', '--fail-under', '100', '--include', 'certbot/*', '--show-missing']' returned non-zero exit status 2 Test coverage on certbot did not meet threshold of 100%. ERROR: InvocationError for command /Users/bmw/Development/certbot/certbot/.tox/py27-cover/bin/python tox.cover.py (exited with code 1) _________________________________________________________________________________________________________________________________________________________ summary _________________________________________________________________________________________________________________________________________________________ ERROR: py27-cover: commands failed ``` I printed the exception just so we're not throwing away information. I think it's also possible we fail for a reason other than the threshold not meeting the percentage, but I've personally never seen this, `coverage report` output is not being captured so hopefully that would inform devs if something else is going on, and saying something like "Test coverage probably did not..." seems like overkill to me personally. * remove codecov * remove unused variable group * remove codecov.yml * Improve tox.cover.py failure output. --- .azure-pipelines/templates/tests-suite.yml | 13 ------------- .codecov.yml | 18 ------------------ .travis.yml | 4 +--- certbot/README.rst | 6 +----- tools/dev_constraints.txt | 1 - tox.cover.py | 19 +++++++++++++------ 6 files changed, 15 insertions(+), 46 deletions(-) delete mode 100644 .codecov.yml diff --git a/.azure-pipelines/templates/tests-suite.yml b/.azure-pipelines/templates/tests-suite.yml index 069ea94d6..d330b7954 100644 --- a/.azure-pipelines/templates/tests-suite.yml +++ b/.azure-pipelines/templates/tests-suite.yml @@ -25,8 +25,6 @@ jobs: PYTEST_ADDOPTS: --numprocesses 4 pool: vmImage: $(IMAGE_NAME) - variables: - - group: certbot-common steps: - bash: brew install augeas condition: startswith(variables['IMAGE_NAME'], 'macOS') @@ -39,14 +37,3 @@ jobs: displayName: Install dependencies - script: python -m tox displayName: Run tox - # We do not require codecov report upload to succeed. So to avoid to break the pipeline if - # something goes wrong, each command is suffixed with a command that hides any non zero exit - # codes and echoes an informative message instead. - - bash: | - curl -s https://codecov.io/bash -o codecov-bash || echo "Failed to download codecov-bash" - chmod +x codecov-bash || echo "Failed to apply execute permissions on codecov-bash" - ./codecov-bash -F windows || echo "Codecov did not collect coverage reports" - condition: in(variables['TOXENV'], 'py37-cover', 'integration-certbot') - env: - CODECOV_TOKEN: $(codecov_token) - displayName: Publish coverage diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 0a97fffe3..000000000 --- a/.codecov.yml +++ /dev/null @@ -1,18 +0,0 @@ -coverage: - status: - project: - default: off - linux: - flags: linux - # Fixed target instead of auto set by #7173, can - # be removed when flags in Codecov are added back. - target: 97.4 - threshold: 0.1 - base: auto - windows: - flags: windows - # Fixed target instead of auto set by #7173, can - # be removed when flags in Codecov are added back. - target: 97.4 - threshold: 0.1 - base: auto diff --git a/.travis.yml b/.travis.yml index e5354898d..d498d0305 100644 --- a/.travis.yml +++ b/.travis.yml @@ -247,15 +247,13 @@ addons: # version of virtualenv. The option "-I" is set so when CERTBOT_NO_PIN is also # set, pip updates dependencies it thinks are already satisfied to avoid some # problems with its lack of real dependency resolution. -install: 'tools/pip_install.py -I codecov tox virtualenv' +install: 'tools/pip_install.py -I tox virtualenv' # Most of the time TRAVIS_RETRY is an empty string, and has no effect on the # script command. It is set only to `travis_retry` during farm tests, in # order to trigger the Travis retry feature, and compensate the inherent # flakiness of these specific tests. script: '$TRAVIS_RETRY tox' -after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux' - notifications: email: false irc: diff --git a/certbot/README.rst b/certbot/README.rst index d1b1e4fe2..5ed74f247 100644 --- a/certbot/README.rst +++ b/certbot/README.rst @@ -71,16 +71,12 @@ ACME spec: http://ietf-wg-acme.github.io/acme/ ACME working area in github: https://github.com/ietf-wg-acme/acme -|build-status| |coverage| |container| +|build-status| |container| .. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master :target: https://travis-ci.com/certbot/certbot :alt: Travis CI status -.. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg - :target: https://codecov.io/gh/certbot/certbot - :alt: Coverage status - .. |container| image:: https://quay.io/repository/letsencrypt/letsencrypt/status :target: https://quay.io/repository/letsencrypt/letsencrypt :alt: Docker Repository on Quay.io diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 7d2013c7a..cfa036435 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -18,7 +18,6 @@ boto3==1.11.7 botocore==1.14.7 cached-property==1.5.1 cloudflare==2.3.1 -codecov==2.0.15 configparser==3.7.4 contextlib2==0.6.0.post1 coverage==4.5.4 diff --git a/tox.cover.py b/tox.cover.py index 3e69a14d6..4848b2740 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function + import argparse import os import subprocess @@ -48,18 +50,23 @@ def cover(package): subprocess.check_call([sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=', pkg_dir]) - subprocess.check_call([ - sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', - '{0}/*'.format(pkg_dir), '--show-missing']) + try: + subprocess.check_call([ + sys.executable, '-m', 'coverage', 'report', '--fail-under', + str(threshold), '--include', '{0}/*'.format(pkg_dir), + '--show-missing']) + except subprocess.CalledProcessError as err: + print(err) + print('Test coverage on', pkg_dir, + 'did not meet threshold of {0}%.'.format(threshold)) + sys.exit(1) def main(): description = """ This script is used by tox.ini (and thus by Travis CI and Azure Pipelines) in order to generate separate stats for each package. It should be removed once -those packages are moved to a separate repo. - -Option -e makes sure we fail fast and don't submit to codecov.""" +those packages are moved to a separate repo.""" parser = argparse.ArgumentParser(description=description) parser.add_argument('--packages', nargs='+') From 50ea6085537dfec3bceaa4f9f4e4065de84d1407 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 Feb 2020 15:07:33 -0800 Subject: [PATCH 44/92] Don't run advanced tests on PRs. (#7820) When I wrote https://github.com/certbot/certbot/pull/7813, I didn't understand the default behavior for pull requests if you don't specify `pr` in the yaml file. According to https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=azure-devops&tabs=yaml#pr-triggers: > If no pr triggers appear in your YAML file, pull request builds are automatically enabled for all branches... This is not the behavior we want. This PR fixes the problem by disabling builds on PRs. You should be able to see this working because the advanced tests should not run on this PR but they did run on https://github.com/certbot/certbot/pull/7811. --- .azure-pipelines/advanced-test.yml | 1 + .azure-pipelines/advanced.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.azure-pipelines/advanced-test.yml b/.azure-pipelines/advanced-test.yml index b9ac9c38a..5be29ba79 100644 --- a/.azure-pipelines/advanced-test.yml +++ b/.azure-pipelines/advanced-test.yml @@ -4,6 +4,7 @@ trigger: # "Running tests in CI" is still correct. - azure-test-* - test-* +pr: none jobs: # Any addition here should be reflected in the advanced and release pipelines. diff --git a/.azure-pipelines/advanced.yml b/.azure-pipelines/advanced.yml index 7f0f5de50..d950e6524 100644 --- a/.azure-pipelines/advanced.yml +++ b/.azure-pipelines/advanced.yml @@ -1,6 +1,7 @@ # Advanced pipeline for running our full test suite on protected branches. trigger: - '*.x' +pr: none # This pipeline is also nightly run on master schedules: - cron: "0 4 * * *" From 9f8e4507ad0cb3dbedb726dda4c46affb1eb7ad3 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Fri, 28 Feb 2020 00:44:23 +0000 Subject: [PATCH 45/92] Document safe and simple usage by services without root privileges (#7821) Certificates are public information by design: they are provided by web servers without any prior authentication required. In a public key cryptographic system, only the private key is secret information. The private key file is already created as accessible only to the root user with mode 0600, and these file permissions are set before any key content is written to the file. There is no window within which an attacker with access to the containing directory would be able to read the private key content. Older versions of Certbot (prior to 0.29.0) would create private key files with mode 0644 and rely solely on the containing directory permissions to restrict access. We therefore cannot (yet) set the relevant default directory permissions to 0755, since it is possible that a user could install Certbot, obtain a certificate, then downgrade to a pre-0.29.0 version of Certbot, then obtain another certificate. This chain of events would leave the second certificate's private key file exposed. As a compromise solution, document the fact that it is safe for the common case of non-downgrading users to change the permissions of /etc/letsencrypt/{live,archive} to 0755, and explain how to use chgrp and chmod to make the private key file readable by a non-root service user. This provides guidance on the simplest way to solve the common problem of making keys and certificates usable by services that run without root privileges, with no requirement to create a custom (and hence error-prone) executable hook. Remove the existing custom executable hook example, so that the documentation contains only the simplest and safest way to solve this very common problem. Signed-off-by: Michael Brown --- certbot/docs/using.rst | 48 ++++++++++-------------------------------- 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 27ae826bd..8ec172c24 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -485,43 +485,6 @@ If you want your hook to run only after a successful renewal, use ``certbot renew --deploy-hook /path/to/deploy-hook-script`` -For example, if you have a daemon that does not read its certificates as the -root user, a deploy hook like this can copy them to the correct location and -apply appropriate file permissions. - -/path/to/deploy-hook-script - -.. code-block:: none - - #!/bin/sh - - set -e - - for domain in $RENEWED_DOMAINS; do - case $domain in - example.com) - daemon_cert_root=/etc/some-daemon/certs - - # Make sure the certificate and private key files are - # never world readable, even just for an instant while - # we're copying them into daemon_cert_root. - umask 077 - - cp "$RENEWED_LINEAGE/fullchain.pem" "$daemon_cert_root/$domain.cert" - cp "$RENEWED_LINEAGE/privkey.pem" "$daemon_cert_root/$domain.key" - - # Apply the proper file ownership and permissions for - # the daemon to read its certificate and key. - chown some-daemon "$daemon_cert_root/$domain.cert" \ - "$daemon_cert_root/$domain.key" - chmod 400 "$daemon_cert_root/$domain.cert" \ - "$daemon_cert_root/$domain.key" - - service some-daemon restart >/dev/null - ;; - esac - done - You can also specify hooks by placing files in subdirectories of Certbot's configuration directory. Assuming your configuration directory is ``/etc/letsencrypt``, any executable files found in @@ -686,6 +649,17 @@ your (web) server configuration directly to those files (or create symlinks). During the renewal_, ``/etc/letsencrypt/live`` is updated with the latest necessary files. +For historical reasons, the containing directories are created with +permissions of ``0700`` meaning that certificates are accessible only +to servers that run as the root user. **If you will never downgrade +to an older version of Certbot**, then you can safely fix this using +``chmod 0755 /etc/letsencrypt/{live,archive}``. + +For servers that drop root privileges before attempting to read the +private key file, you will also need to use ``chgrp`` and ``chmod +0640`` to allow the server to read +``/etc/letsencrypt/live/$domain/privkey.pem``. + .. note:: ``/etc/letsencrypt/archive`` and ``/etc/letsencrypt/keys`` contain all previous keys and certificates, while ``/etc/letsencrypt/live`` symlinks to the latest versions. From 31470262110ab8e9a388d738461b08a4f489d430 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Mar 2020 11:07:15 -0800 Subject: [PATCH 46/92] Check OCSP as part of determining if the certificate is due for renewal (#7829) Fixes #1028. Doing this now because of https://community.letsencrypt.org/t/revoking-certain-certificates-on-march-4/. The new `ocsp_revoked_by_paths` function is taken from https://github.com/certbot/certbot/pull/7649 with the optional argument removed for now because it is unused. This function was added in this PR because `storage.py` uses `self.latest_common_version()` to determine which certificate should be looked at for determining renewal status at https://github.com/certbot/certbot/blob/9f8e4507ad0cb3dbedb726dda4c46affb1eb7ad3/certbot/certbot/_internal/storage.py#L939-L947 I think this is unnecessary and you can just look at the currently linked certificate, but I don't think we should be changing the logic that code has always had now. * Check OCSP status as part of determining to renew * add integration tests * add ocsp_revoked_by_paths --- .../certbot_tests/test_main.py | 17 ++++++++++ certbot/CHANGELOG.md | 2 ++ certbot/certbot/_internal/storage.py | 33 +++++++++++-------- certbot/certbot/ocsp.py | 13 +++++++- certbot/tests/storage_test.py | 33 ++++++++++++++++--- 5 files changed, 80 insertions(+), 18 deletions(-) diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 94e76cf79..f0c5edd3f 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -595,6 +595,23 @@ def test_ocsp_status_live(context): assert output.count('REVOKED') == 1, 'Expected {0} to be REVOKED'.format(cert) +def test_ocsp_renew(context): + """Test that revoked certificates are renewed.""" + # Obtain a certificate + certname = context.get_domain('ocsp-renew') + context.certbot(['--domains', certname]) + + # Test that "certbot renew" does not renew the certificate + assert_cert_count_for_lineage(context.config_dir, certname, 1) + context.certbot(['renew'], force_renew=False) + assert_cert_count_for_lineage(context.config_dir, certname, 1) + + # Revoke the certificate and test that it does renew the certificate + context.certbot(['revoke', '--cert-name', certname, '--no-delete-after-revoke']) + context.certbot(['renew'], force_renew=False) + assert_cert_count_for_lineage(context.config_dir, certname, 2) + + def test_dry_run_deactivate_authzs(context): """Test that Certbot deactivates authorizations when performing a dry run""" diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 6c1b112d7..2a934ee5b 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -13,6 +13,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed +* Certbot will now renew certificates early if they have been revoked according + to OCSP. * Fix acme module warnings when response Content-Type includes params (e.g. charset). * Fixed issue where webroot plugin would incorrectly raise `Read-only file system` error when creating challenge directories (issue #7165). diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 6a34355a8..2dac163e2 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -15,6 +15,7 @@ import certbot from certbot import crypto_util from certbot import errors from certbot import interfaces +from certbot import ocsp from certbot import util from certbot._internal import cli from certbot._internal import constants @@ -882,27 +883,33 @@ class RenewableCert(interfaces.RenewableCert): with open(target) as f: return crypto_util.get_names_from_cert(f.read()) - def ocsp_revoked(self, version=None): - # pylint: disable=unused-argument + def ocsp_revoked(self, version): """Is the specified cert version revoked according to OCSP? - Also returns True if the cert version is declared as intended - to be revoked according to Let's Encrypt OCSP extensions. - (If no version is specified, uses the current version.) - - This method is not yet implemented and currently always returns - False. + Also returns True if the cert version is declared as revoked + according to OCSP. If OCSP status could not be determined, False + is returned. :param int version: the desired version number - :returns: whether the certificate is or will be revoked + :returns: True if the certificate is revoked, otherwise, False :rtype: bool """ - # XXX: This query and its associated network service aren't - # implemented yet, so we currently return False (indicating that the - # certificate is not revoked). - return False + cert_path = self.version("cert", version) + chain_path = self.version("chain", version) + # While the RevocationChecker should return False if it failed to + # determine the OCSP status, let's ensure we don't crash Certbot by + # catching all exceptions here. + try: + return ocsp.RevocationChecker().ocsp_revoked_by_paths(cert_path, + chain_path) + except Exception as e: # pylint: disable=broad-except + logger.warning( + "An error occurred determining the OCSP status of %s.", + cert_path) + logger.debug(str(e)) + return False def autorenewal_is_enabled(self): """Is automatic renewal enabled for this cert? diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 6a95f26fa..9799c675c 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -68,8 +68,19 @@ class RevocationChecker(object): :rtype: bool """ - cert_path, chain_path = cert.cert_path, cert.chain_path + return self.ocsp_revoked_by_paths(cert.cert_path, cert.chain_path) + def ocsp_revoked_by_paths(self, cert_path, chain_path): + # type: (str, str) -> bool + """Performs the OCSP revocation check + + :param str cert_path: Certificate filepath + :param str chain_path: Certificate chain filepath + + :returns: True if revoked; False if valid or the check failed or cert is expired. + :rtype: bool + + """ if self.broken: return False diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 6208974ec..0f7620b78 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -672,10 +672,35 @@ class RenewableCertTests(BaseRenewableCertTest): errors.CertStorageError, self.test_rc._update_link_to, "elephant", 17) - def test_ocsp_revoked(self): - # XXX: This is currently hardcoded to False due to a lack of an - # OCSP server to test against. - self.assertFalse(self.test_rc.ocsp_revoked()) + @mock.patch("certbot.ocsp.RevocationChecker.ocsp_revoked_by_paths") + def test_ocsp_revoked(self, mock_checker): + # Write out test files + for kind in ALL_FOUR: + self._write_out_kind(kind, 1) + version = self.test_rc.latest_common_version() + expected_cert_path = self.test_rc.version("cert", version) + expected_chain_path = self.test_rc.version("chain", version) + + # Test with cert revoked + mock_checker.return_value = True + self.assertTrue(self.test_rc.ocsp_revoked(version)) + self.assertEqual(mock_checker.call_args[0][0], expected_cert_path) + self.assertEqual(mock_checker.call_args[0][1], expected_chain_path) + + # Test with cert not revoked + mock_checker.return_value = False + self.assertFalse(self.test_rc.ocsp_revoked(version)) + self.assertEqual(mock_checker.call_args[0][0], expected_cert_path) + self.assertEqual(mock_checker.call_args[0][1], expected_chain_path) + + # Test with error + mock_checker.side_effect = ValueError + with mock.patch("certbot._internal.storage.logger.warning") as logger: + self.assertFalse(self.test_rc.ocsp_revoked(version)) + self.assertEqual(mock_checker.call_args[0][0], expected_cert_path) + self.assertEqual(mock_checker.call_args[0][1], expected_chain_path) + log_msg = logger.call_args[0][0] + self.assertIn("An error occurred determining the OCSP status", log_msg) def test_add_time_interval(self): from certbot._internal import storage From b1fb3296e949c5ce5175321328a89a41b7dd3d12 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Mar 2020 12:36:36 -0800 Subject: [PATCH 47/92] Update changelog for 1.3.0 release --- certbot/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 2a934ee5b..493e4d5c1 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.3.0 - master +## 1.3.0 - 2020-03-03 ### Added From 6edb4e1a3924821316b9344adb9c533937426fa7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Mar 2020 12:43:02 -0800 Subject: [PATCH 48/92] Release 1.3.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 26 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- certbot/docs/cli-help.txt | 2 +- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 26 files changed, 79 insertions(+), 79 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 0e11779ba..922cae26c 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index b37ee3972..6483313b8 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index cea58e2cb..0ea3275c3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.2.0" +LE_AUTO_VERSION="1.3.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1540,18 +1540,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.2.0 \ - --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \ - --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c -acme==1.2.0 \ - --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \ - --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab -certbot-apache==1.2.0 \ - --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \ - --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3 -certbot-nginx==1.2.0 \ - --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \ - --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db +certbot==1.3.0 \ + --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ + --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 +acme==1.3.0 \ + --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ + --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d +certbot-apache==1.3.0 \ + --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ + --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 +certbot-nginx==1.3.0 \ + --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ + --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 1dbcefa75..f85311999 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.3.0.dev0' +version = '1.3.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 9376bc1c4..d5da3ed95 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 4e99ff5ff..491096d70 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 9c9d1717c..23c2903f4 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 9cde6214c..197873733 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index adaba6851..b0a016441 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index a849cef45..2c76869bc 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 51d5b8a3f..61528b3a0 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index e7e91b929..1e7829588 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index ea64f79a2..418c635b9 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index d6bedca1c..99bc1022e 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 8f5b052a2..9f5974bcb 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index fa51c2108..42c6f11bd 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index f25e348ff..64e307b52 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 8df2320ba..b18da9d97 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b180fe06a..fe707fb09 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.3.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 84ade6b08..39171bad2 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.3.0.dev0' +__version__ = '1.3.0' diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index ff49609c4..3c2289030 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -113,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.2.0 (certbot(-auto); + "". (default: CertbotACMEClient/1.3.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the user agent are: --duplicate, diff --git a/letsencrypt-auto b/letsencrypt-auto index cea58e2cb..0ea3275c3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.2.0" +LE_AUTO_VERSION="1.3.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1540,18 +1540,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.2.0 \ - --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \ - --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c -acme==1.2.0 \ - --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \ - --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab -certbot-apache==1.2.0 \ - --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \ - --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3 -certbot-nginx==1.2.0 \ - --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \ - --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db +certbot==1.3.0 \ + --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ + --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 +acme==1.3.0 \ + --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ + --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d +certbot-apache==1.3.0 \ + --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ + --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 +certbot-nginx==1.3.0 \ + --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ + --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 488d0bf2e..84473dc30 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl456ZoACgkQTRfJlc2X -dfJx8wf/addMw4kUlwu6poHqLvsifZzHAESgvq+qybgFvl5yTh2U+99PGBgxRYx+ -bENIWBi6+XB+CiVuLzIXWw/VkXh+za99orRkkVK9PI33Xr7jBMZo5Oa3JviYjl3X -PcfjioRQCD+a9Tf9RO25LXQmxn87Ql9x3nxJuk//YeSpuImFmYjIBPE4n/LPEf7z -8WHU4oxxa/bgqGCPgv6O7ZBw7ipd3g+VHcDZcNQMP4tWYb6m7x/nN61yirid7q3M -uqQ1lbitN48ISyru6xPyE6WGTvfl1SIQd21FNRETpcoesx+MTv3ApWT4dqXjZvaX -FeM55IS65e7ci6yLV9qdAbqGKzhX0Q== -=uLcV +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl5ewVUACgkQTRfJlc2X +dfJnZAf+KmxYl1YoP/FlTG5Npb64qaDdxm59SeEVJez6fZh15xq71tRPYR+4xszE +XTeyGt7uAxjYqeiBJU5xBvGC1Veprhj5AbflVOTP+5yiBr9iNWC35zmgaE63UlZ/ +V94sfL0pkax7wLngil7a0OuzUjikzK3gXOqrY8LoUdr4mAA9AhSjajWHmyY3tpDR +84GKrVhybIt0sjy/172VuPPbXZKno/clztkKMZHXNrDeL5jgJ15Va4Ts5FK0j9VT +HQvuazbGkYVCuvlp8Np5ESDje69LCJfPZxl34htoa8WNJoVIOsQWZpoXp5B5huSP +vGrh4LabZ5UDsl+k11ikHBRUpO7E5w== +=IgRH -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index e2813853b..0ea3275c3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.3.0.dev0" +LE_AUTO_VERSION="1.3.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1540,18 +1540,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.2.0 \ - --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \ - --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c -acme==1.2.0 \ - --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \ - --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab -certbot-apache==1.2.0 \ - --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \ - --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3 -certbot-nginx==1.2.0 \ - --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \ - --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db +certbot==1.3.0 \ + --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ + --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 +acme==1.3.0 \ + --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ + --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d +certbot-apache==1.3.0 \ + --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ + --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 +certbot-nginx==1.3.0 \ + --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ + --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index fefc81b37796cdecd066b9bb212d8e285fffd4d0..8c4f52d6e2992d76012ed4c5e8b65c366e53cf8c 100644 GIT binary patch literal 256 zcmV+b0ssEUOnRo|t9S`PT@Q6pdEx5)TLaw|tXK(Czj^_>8PhYd%g<>HQ?bGidlv-5 zE}9A&SzM6uK*eeB!}iF{EuInepHwl!!N=ypH>Giu0~gNuz3H_G-xK$*4GGCrIi8a} zc|vYH^EA>Vt1;uU?ETR7;K}DFE1nDBiU-xb=ODbJ#yW{#gBQmTy?oXA)L zay!@Hc;qXdB0+{NAC2*m>{YND9$B?_{$Q;DGoOCVmMhh`)kRLNXuPfjru6g4?PpB5 zE=0kjQwl0Y1m+t}(iNS8e1OH8)e$OZ#|vb^J4)NEd(wxnV%*phX+ZJIy2kBPS7%B+ G&t7>eJc8N) literal 256 zcmV+b0ssCyy6+%Byu~4aOZx+sGM?uWzM?nz3Jfz_Cb^!H(J$u3EbYx+Wmt@nUdJdC z5^ed_c;i%mwYmiD>ud)5-Lk6YWkh7j-?{0%L?KJZd%TEUG+|;|-*#FI(%jlkvk;lO zO~Yh(rw(~6mZiF)`O}!O(DYn@gPBFqESJ@M(%s)|7%F?plRgOU1Sm{BW(R9Bfx|N( zKD3y8K_?ev`Hu@xHbPDe3A34WZ3?o_F0i8u)P>>`q{3hr9CRb?E6FEIK^btkZ@Qg6 zt+z9SQGsl`GlM4_;VnLX96Xg&)Kv+HS#^V0%aEYWG9N+LKk6WSdEBew*pUM_Rk_eZ GIvjh!?|N Date: Tue, 3 Mar 2020 12:43:03 -0800 Subject: [PATCH 49/92] Add contents to certbot/CHANGELOG.md for next version --- certbot/CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 493e4d5c1..cb1f2968c 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,6 +2,22 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 1.4.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + ## 1.3.0 - 2020-03-03 ### Added From 144d4f2b446a5659e713133868eb921885ce105b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 3 Mar 2020 12:43:04 -0800 Subject: [PATCH 50/92] Bump version to 1.4.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 922cae26c..0527b3fb5 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 6483313b8..4ec1d0a9c 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index f85311999..d6760576a 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.3.0' +version = '1.4.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d5da3ed95..67aac3231 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 491096d70..6c653967d 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 23c2903f4..c45fc8d03 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 197873733..9124f0552 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index b0a016441..2a4fd92b0 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 2c76869bc..2198fdd3e 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 61528b3a0..087766edd 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 1e7829588..1e6b96b71 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 418c635b9..4db50b56c 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 99bc1022e..49e5e3bcf 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 9f5974bcb..6c66b39dc 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 42c6f11bd..5e0900e4d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 64e307b52..98455c362 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index b18da9d97..16990a4d3 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index fe707fb09..34785f963 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 39171bad2..0ce7ff6b7 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.3.0' +__version__ = '1.4.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0ea3275c3..ca0bda2d5 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.3.0" +LE_AUTO_VERSION="1.4.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From d72a1a71d22792cecf66a8d636705b9319e8fca3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 5 Mar 2020 11:50:52 -0800 Subject: [PATCH 51/92] Fix issues with Azure Pipelines (#7838) This PR fixes two issues. First, it fixes #7814 by removing our tests on Windows Server 2012. I also added the sentence "Certbot supports Windows Server 2016 and Windows Server 2019." to https://community.letsencrypt.org/t/beta-phase-of-certbot-for-windows/105822. Second, it fixes the test failures which can be seen at https://dev.azure.com/certbot/certbot/_build/results?buildId=1309&view=results by no longer manually installing our own version of Python and instead using the one provided by Azure. These small changes are in the same PR because I wanted to fix test failures ASAP and `UsePythonVersion` is not available on Windows 2012. See https://github.com/certbot/certbot/pull/7641#discussion_r358510854. You can see tests passing with this change at https://dev.azure.com/certbot/certbot/_build/results?buildId=1311&view=results. * stop testing on win2012 * switch to UsePythonVersion --- .azure-pipelines/templates/installer-tests.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/templates/installer-tests.yml b/.azure-pipelines/templates/installer-tests.yml index 6d5672339..ea2101792 100644 --- a/.azure-pipelines/templates/installer-tests.yml +++ b/.azure-pipelines/templates/installer-tests.yml @@ -28,15 +28,13 @@ jobs: imageName: windows-2019 win2016: imageName: vs2017-win2016 - win2012r2: - imageName: vs2015-win2012r2 pool: vmImage: $(imageName) steps: - - powershell: Invoke-WebRequest https://www.python.org/ftp/python/3.8.1/python-3.8.1-amd64-webinstall.exe -OutFile C:\py3-setup.exe - displayName: Get Python - - script: C:\py3-setup.exe /quiet PrependPath=1 InstallAllUsers=1 Include_launcher=1 InstallLauncherAllUsers=1 Include_test=0 Include_doc=0 Include_dev=1 Include_debug=0 Include_tcltk=0 TargetDir=C:\py3 - displayName: Install Python + - task: UsePythonVersion@0 + inputs: + versionSpec: 3.8 + addToPath: true - task: DownloadPipelineArtifact@2 inputs: artifact: windows-installer From 7f63141e410ae7ce200c7c7408d05c87eae0c5f5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 6 Mar 2020 09:46:30 -0800 Subject: [PATCH 52/92] Add changes to the correct changelog entry (#7833) https://github.com/certbot/certbot/pull/7742 and https://github.com/certbot/certbot/pull/7738 landed after our 1.2.0 release, but the 1.2.0 changelog entry was modified instead of the one for master/1.3.0. This PR moves the changelog entries to the 1.3.0 section. --- certbot/CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index cb1f2968c..868f3c8be 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -26,6 +26,7 @@ More details about these changes can be found on our GitHub repo. determine the OCSP status of certificates. * Don't verify the existing certificate in HTTP01Response.simple_verify, for compatibility with the real-world ACME challenge checks. +* Added support for `$hostname` in nginx `server_name` directive ### Changed @@ -37,7 +38,7 @@ More details about these changes can be found on our GitHub repo. ### Fixed -* +* Fix Apache plugin to use less restrictive umask for making the challenge directory when a restrictive umask was set when certbot was started. More details about these changes can be found on our GitHub repo. @@ -46,7 +47,6 @@ More details about these changes can be found on our GitHub repo. ### Added * Added support for Cloudflare's limited-scope API Tokens -* Added support for `$hostname` in nginx `server_name` directive ### Changed @@ -59,7 +59,6 @@ More details about these changes can be found on our GitHub repo. ### Fixed * Fix collections.abc imports for Python 3.9. -* Fix Apache plugin to use less restrictive umask for making the challenge directory when a restrictive umask was set when certbot was started. More details about these changes can be found on our GitHub repo. From 69aec55ead42c9ef9231602d002825e838e9dafb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 9 Mar 2020 13:05:35 -0700 Subject: [PATCH 53/92] Remove --no-site-packages outside of certbot-auto. (#7832) --- certbot-compatibility-test/Dockerfile | 2 +- tools/_release.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index a9996f779..a6a0c93db 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -31,7 +31,7 @@ COPY certbot-nginx /opt/certbot/src/certbot-nginx/ COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ COPY tools /opt/certbot/src/tools -RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ +RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv -p python2 /opt/certbot/venv && \ /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install -U pip ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/tools/_release.sh b/tools/_release.sh index 1819adad2..97d5f5eb8 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -59,7 +59,7 @@ mv "dist.$version" "dist.$version.$(date +%s).bak" || true git tag --delete "$tag" || true tmpvenv=$(mktemp -d) -VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 $tmpvenv +VIRTUALENV_NO_DOWNLOAD=1 virtualenv -p python2 $tmpvenv . $tmpvenv/bin/activate # update setuptools/pip just like in other places in the repo pip install -U setuptools @@ -160,7 +160,7 @@ cd "dist.$version" python -m SimpleHTTPServer $PORT & # cd .. is NOT done on purpose: we make sure that all subpackages are # installed from local PyPI rather than current directory (repo root) -VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages ../venv +VIRTUALENV_NO_DOWNLOAD=1 virtualenv ../venv . ../venv/bin/activate pip install -U setuptools pip install -U pip From 78168a5248cf119289053ffcf048c4be9b2af9d6 Mon Sep 17 00:00:00 2001 From: radek-sprta Date: Wed, 11 Mar 2020 21:27:19 +0100 Subject: [PATCH 54/92] Add CloudDNS to third-party plugins (#7840) --- certbot/docs/using.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 8ec172c24..3c3ef6fad 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -280,6 +280,7 @@ pritunl_ N Y Install certificates in pritunl distributed OpenVPN proxmox_ N Y Install certificates in Proxmox Virtualization servers dns-standalone_ Y N Obtain certificates via an integrated DNS server dns-ispconfig_ Y N DNS Authentication using ISPConfig as DNS server +dns-clouddns_ Y N DNS Authentication using CloudDNS API ================== ==== ==== =============================================================== .. _haproxy: https://github.com/greenhost/certbot-haproxy @@ -291,6 +292,7 @@ dns-ispconfig_ Y N DNS Authentication using ISPConfig as DNS server .. _external-auth: https://github.com/EnigmaBridge/certbot-external-auth .. _dns-standalone: https://github.com/siilike/certbot-dns-standalone .. _dns-ispconfig: https://github.com/m42e/certbot-dns-ispconfig +.. _dns-clouddns: https://github.com/vshosting/certbot-dns-clouddns If you're interested, you can also :ref:`write your own plugin `. From 44b97df4e91a3d228bf933ee169e964070f96dd3 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 12 Mar 2020 17:29:03 +0100 Subject: [PATCH 55/92] Exposes environment variable to let hooks scripts know when the last challenge is handled (#7837) Fixes #5484 This PRs makes Certbot expose two new environment variables in the auth and cleanup hooks of the `manual` plugin: * `CERTBOT_REMAINING_CHALLENGES` contains the number of challenges that remain after the current one (so it equals to 0 when the script is called for the last challenge) * `CERTBOT_ALL_DOMAINS` contains a comma-separated list of all domains concerned by a challenge for the current certificate With these variables, an hook script can know when it is run for the last time, and then trigger appropriate finalizers for all challenges that have been executed. This will be particularly useful for certificates with a lot of domains validated with DNS-01 challenges: instead of waiting on each hook execution to check that the relevant DNS TXT entry has been inserted, these waits can be avoided thanks to the latest hook verifying all domains in one run. * Inject environment variables in manual scripts about remaining challenges * Adapt tests * Less variables and less lines * Update manual.py * Update manual_test.py * Add documentation * Add changelog --- certbot/CHANGELOG.md | 5 ++++- certbot/certbot/_internal/plugins/manual.py | 21 +++++++++++++-------- certbot/docs/using.rst | 4 +++- certbot/tests/plugins/manual_test.py | 19 +++++++++++++------ 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 868f3c8be..903af0610 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -6,7 +6,10 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added -* +* Expose two new environment variables in the authenticator and cleanup scripts used by + the `manual` plugin: `CERTBOT_REMAINING_CHALLENGES` is equal to the number of challenges + remaining after the current challenge, `CERTBOT_ALL_DOMAINS` is a comma-separated list + of all domains challenged for the current certificate. ### Changed diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index 3204fe1da..87ccdbd7e 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -35,7 +35,11 @@ class Authenticator(common.Plugin): 'is the validation string, and $CERTBOT_TOKEN is the filename of the ' 'resource requested when performing an HTTP-01 challenge. An additional ' 'cleanup script can also be provided and can use the additional variable ' - '$CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth script.') + '$CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth script.' + 'For both authenticator and cleanup script, on HTTP-01 and DNS-01 challenges,' + '$CERTBOT_REMAINING_CHALLENGES will be equal to the number of challenges that ' + 'remain after the current one, and $CERTBOT_ALL_DOMAINS contains a comma-separated ' + 'list of all domains that are challenged for the current certificate.') _DNS_INSTRUCTIONS = """\ Please deploy a DNS TXT record under the name {domain} with the following value: @@ -109,14 +113,13 @@ permitted by DNS standards.) def perform(self, achalls): # pylint: disable=missing-function-docstring self._verify_ip_logging_ok() - if self.conf('auth-hook'): - perform_achall = self._perform_achall_with_script - else: - perform_achall = self._perform_achall_manually responses = [] for achall in achalls: - perform_achall(achall) + if self.conf('auth-hook'): + self._perform_achall_with_script(achall, achalls) + else: + self._perform_achall_manually(achall) responses.append(achall.response(achall.account_key)) return responses @@ -134,9 +137,11 @@ permitted by DNS standards.) else: raise errors.PluginError('Must agree to IP logging to proceed') - def _perform_achall_with_script(self, achall): + def _perform_achall_with_script(self, achall, achalls): env = dict(CERTBOT_DOMAIN=achall.domain, - CERTBOT_VALIDATION=achall.validation(achall.account_key)) + CERTBOT_VALIDATION=achall.validation(achall.account_key), + CERTBOT_ALL_DOMAINS=','.join(one_achall.domain for one_achall in achalls), + CERTBOT_REMAINING_CHALLENGES=str(len(achalls) - achalls.index(achall) - 1)) if isinstance(achall.chall, challenges.HTTP01): env['CERTBOT_TOKEN'] = achall.chall.encode('token') else: diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 3c3ef6fad..d3c2d1582 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -738,8 +738,10 @@ the ``cleanup.sh`` script. Additionally certbot will pass relevant environment variables to these scripts: - ``CERTBOT_DOMAIN``: The domain being authenticated -- ``CERTBOT_VALIDATION``: The validation string (HTTP-01 and DNS-01 only) +- ``CERTBOT_VALIDATION``: The validation string - ``CERTBOT_TOKEN``: Resource name part of the HTTP-01 challenge (HTTP-01 only) +- ``CERTBOT_REMAINING_CHALLENGES``: Number of challenges remaining after the current challenge +- ``CERTBOT_ALL_DOMAINS``: A comma-separated list of all domains challenged for the current certificate Additionally for cleanup: diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index bd11a9538..6cdef148a 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -72,16 +72,23 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.config.manual_public_ip_logging_ok = True self.config.manual_auth_hook = ( '{0} -c "from __future__ import print_function;' - 'from certbot.compat import os; print(os.environ.get(\'CERTBOT_DOMAIN\'));' + 'from certbot.compat import os;' + 'print(os.environ.get(\'CERTBOT_DOMAIN\'));' 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' - 'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));"' + 'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));' + 'print(os.environ.get(\'CERTBOT_ALL_DOMAINS\'));' + 'print(os.environ.get(\'CERTBOT_REMAINING_CHALLENGES\'));"' .format(sys.executable)) - dns_expected = '{0}\n{1}\n{2}'.format( + dns_expected = '{0}\n{1}\n{2}\n{3}\n{4}'.format( self.dns_achall.domain, 'notoken', - self.dns_achall.validation(self.dns_achall.account_key)) - http_expected = '{0}\n{1}\n{2}'.format( + self.dns_achall.validation(self.dns_achall.account_key), + ','.join(achall.domain for achall in self.achalls), + len(self.achalls) - self.achalls.index(self.dns_achall) - 1) + http_expected = '{0}\n{1}\n{2}\n{3}\n{4}'.format( self.http_achall.domain, self.http_achall.chall.encode('token'), - self.http_achall.validation(self.http_achall.account_key)) + self.http_achall.validation(self.http_achall.account_key), + ','.join(achall.domain for achall in self.achalls), + len(self.achalls) - self.achalls.index(self.http_achall) - 1) self.assertEqual( self.auth.perform(self.achalls), From 2fd85a4f36c37cd7dfa96f129338c2b6d95dd0d8 Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Thu, 12 Mar 2020 17:37:49 +0100 Subject: [PATCH 56/92] Add serial number to certificates output (#7842) Fixes #7835 I had to mock out `get_serial_from_cert` to keep a test from failing, because `cert_path` was mocked itself in `test_report_human_readable`. Also, I kept the same style for the serial number as the recent Let's Encrypt e-mail: lowercase hexadecimal without a `0x` prefix and without colons every 2 chars. Shouldn't be a problem to change the format if required. --- certbot/CHANGELOG.md | 1 + certbot/certbot/_internal/cert_manager.py | 11 +++++++---- certbot/certbot/crypto_util.py | 14 ++++++++++++++ certbot/tests/cert_manager_test.py | 4 +++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 903af0610..2f22b5204 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -6,6 +6,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added +* Added serial number of certificate to the output of `certbot certificates` * Expose two new environment variables in the authenticator and cleanup scripts used by the `manual` plugin: `CERTBOT_REMAINING_CHALLENGES` is equal to the number of challenges remaining after the current challenge, `CERTBOT_ALL_DOMAINS` is a comma-separated list diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index e6cbd5c2c..2652b3d2c 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -276,12 +276,15 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False): status = "VALID: {0} days".format(diff.days) valid_string = "{0} ({1})".format(cert.target_expiry, status) + serial = format(crypto_util.get_serial_from_cert(cert.cert_path), 'x') certinfo.append(" Certificate Name: {0}\n" - " Domains: {1}\n" - " Expiry Date: {2}\n" - " Certificate Path: {3}\n" - " Private Key Path: {4}".format( + " Serial Number: {1}\n" + " Domains: {2}\n" + " Expiry Date: {3}\n" + " Certificate Path: {4}\n" + " Private Key Path: {5}".format( cert.lineagename, + serial, " ".join(cert.names()), valid_string, cert.fullchain, diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 9136445bc..adb972f24 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -491,3 +491,17 @@ def cert_and_chain_from_fullchain(fullchain_pem): crypto.load_certificate(crypto.FILETYPE_PEM, fullchain_pem)).decode() chain = fullchain_pem[len(cert):].lstrip() return (cert, chain) + +def get_serial_from_cert(cert_path): + """Retrieve the serial number of a certificate from certificate path + + :param str cert_path: path to a cert in PEM format + + :returns: serial number of the certificate + :rtype: int + """ + # pylint: disable=redefined-outer-name + with open(cert_path) as f: + x509 = crypto.load_certificate(crypto.FILETYPE_PEM, + f.read()) + return x509.get_serial_number() diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index eb8005b2b..bea64f09c 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -200,9 +200,11 @@ class CertificatesTest(BaseCertManagerTest): self.assertTrue(mock_utility.called) shutil.rmtree(empty_tempdir) + @mock.patch('certbot.crypto_util.get_serial_from_cert') @mock.patch('certbot._internal.cert_manager.ocsp.RevocationChecker.ocsp_revoked') - def test_report_human_readable(self, mock_revoked): + def test_report_human_readable(self, mock_revoked, mock_serial): mock_revoked.return_value = None + mock_serial.return_value = 1234567890 from certbot._internal import cert_manager import datetime import pytz From 07abe7a8d68961042ee301039dd4da87306cb1a0 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 12 Mar 2020 21:53:19 +0100 Subject: [PATCH 57/92] Reimplement tls-alpn-01 in acme (#6886) This PR is the first part of work described in #6724. It reintroduces the tls-alpn-01 challenge in `acme` module, that was introduced by #5894 and reverted by #6100. The reason it was removed in the past is because some tests showed that with `1.0.2` branch of OpenSSL, the self-signed certificate containing the authorization key is sent to the requester even if the ALPN protocol `acme-tls/1` was not declared as supported by the requester during the TLS handshake. However recent discussions lead to the conclusion that this behavior was not a security issue, because first it is coherent with the behavior with servers that do not support ALPN at all, and second it cannot make a tls-alpn-01 challenge be validated in this kind of corner case. On top of the original modifications given by #5894, I merged the code to be up-to-date with our `master`, and fixed tests to match recent evolution about not displaying the `keyAuthorization` in the deserialized JSON form of an ACME challenge. I also move the logic to verify if ALPN is available on the current system, and so that the tls-alpn-01 challenge can be used, to a dedicated static function `is_available` in `acme.challenge.TLSALPN01`. This function is used in the related tests to skip them, and will be used in the future from Certbot plugins to trigger or not the logic related to tls-alpn-01, depending on the OpenSSL version available to Python. * Reimplement TLS-ALPN-01 challenge and standalone TLS-ALPN server from #5894. * Setup a class method to check if tls-alpn-01 is supported. * Add potential missing parameter in validation for tls-alpn * Improve comments * Make a class private * Handle old versions of openssl that do not terminate the handshake when they should do. * Add changelog * Explicitly close the TLS connection by the book. * Remove unused exception * Fix lint --- acme/acme/challenges.py | 172 ++++++++++++++++++++++++--- acme/acme/crypto_util.py | 63 +++++++--- acme/acme/standalone.py | 56 ++++++++- acme/tests/challenges_test.py | 87 ++++++++++++-- acme/tests/crypto_util_test.py | 16 ++- acme/tests/standalone_test.py | 57 ++++++++- acme/tests/testdata/README | 6 +- acme/tests/testdata/rsa1024_cert.pem | 13 ++ certbot/CHANGELOG.md | 2 + 9 files changed, 425 insertions(+), 47 deletions(-) create mode 100644 acme/tests/testdata/rsa1024_cert.pem diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 39c8d6269..0b112be00 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -1,14 +1,20 @@ """ACME Identifier Validation Challenges.""" import abc +import codecs import functools import hashlib import logging +import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose import requests import six +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 +from OpenSSL import crypto +from acme import crypto_util +from acme import errors from acme import fields logger = logging.getLogger(__name__) @@ -362,29 +368,163 @@ class HTTP01(KeyAuthorizationChallenge): @ChallengeResponse.register class TLSALPN01Response(KeyAuthorizationChallengeResponse): - """ACME TLS-ALPN-01 challenge response. - - This class only allows initiating a TLS-ALPN-01 challenge returned from the - CA. Full support for responding to TLS-ALPN-01 challenges by generating and - serving the expected response certificate is not currently provided. - """ + """ACME tls-alpn-01 challenge response.""" typ = "tls-alpn-01" + PORT = 443 + """Verification port as defined by the protocol. -@Challenge.register + You can override it (e.g. for testing) by passing ``port`` to + `simple_verify`. + + """ + + ID_PE_ACME_IDENTIFIER_V1 = b"1.3.6.1.5.5.7.1.30.1" + ACME_TLS_1_PROTOCOL = "acme-tls/1" + + @property + def h(self): + """Hash value stored in challenge certificate""" + return hashlib.sha256(self.key_authorization.encode('utf-8')).digest() + + def gen_cert(self, domain, key=None, bits=2048): + """Generate tls-alpn-01 certificate. + + :param unicode domain: Domain verified by the challenge. + :param OpenSSL.crypto.PKey key: Optional private key used in + certificate generation. If not provided (``None``), then + fresh key will be generated. + :param int bits: Number of bits for newly generated key. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + if key is None: + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) + + + der_value = b"DER:" + codecs.encode(self.h, 'hex') + acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1, + critical=True, value=der_value) + + return crypto_util.gen_ss_cert(key, [domain], force_san=True, + extensions=[acme_extension]), key + + def probe_cert(self, domain, host=None, port=None): + """Probe tls-alpn-01 challenge certificate. + + :param unicode domain: domain being validated, required. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + """ + if host is None: + host = socket.gethostbyname(domain) + logger.debug('%s resolved to %s', domain, host) + if port is None: + port = self.PORT + + return crypto_util.probe_sni(host=host, port=port, name=domain, + alpn_protocols=[self.ACME_TLS_1_PROTOCOL]) + + def verify_cert(self, domain, cert): + """Verify tls-alpn-01 challenge certificate. + + :param unicode domain: Domain name being validated. + :param OpensSSL.crypto.X509 cert: Challenge certificate. + + :returns: Whether the certificate was successfully verified. + :rtype: bool + + """ + # pylint: disable=protected-access + names = crypto_util._pyopenssl_cert_or_req_all_names(cert) + logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), names) + if len(names) != 1 or names[0].lower() != domain.lower(): + return False + + for i in range(cert.get_extension_count()): + ext = cert.get_extension(i) + # FIXME: assume this is the ACME extension. Currently there is no + # way to get full OID of an unknown extension from pyopenssl. + if ext.get_short_name() == b'UNDEF': + data = ext.get_data() + return data == self.h + + return False + + # pylint: disable=too-many-arguments + def simple_verify(self, chall, domain, account_public_key, + cert=None, host=None, port=None): + """Simple verify. + + Verify ``validation`` using ``account_public_key``, optionally + probe tls-alpn-01 certificate and check using `verify_cert`. + + :param .challenges.TLSALPN01 chall: Corresponding challenge. + :param str domain: Domain name being validated. + :param JWK account_public_key: + :param OpenSSL.crypto.X509 cert: Optional certificate. If not + provided (``None``) certificate will be retrieved using + `probe_cert`. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + + :returns: ``True`` if and only if client's control of the domain has been verified. + :rtype: bool + + """ + if not self.verify(chall, account_public_key): + logger.debug("Verification of key authorization in response failed") + return False + + if cert is None: + try: + cert = self.probe_cert(domain=domain, host=host, port=port) + except errors.Error as error: + logger.debug(str(error), exc_info=True) + return False + + return self.verify_cert(domain, cert) + + +@Challenge.register # pylint: disable=too-many-ancestors class TLSALPN01(KeyAuthorizationChallenge): - """ACME tls-alpn-01 challenge. - - This class simply allows parsing the TLS-ALPN-01 challenge returned from - the CA. Full TLS-ALPN-01 support is not currently provided. - - """ - typ = "tls-alpn-01" + """ACME tls-alpn-01 challenge.""" response_cls = TLSALPN01Response + typ = response_cls.typ def validation(self, account_key, **kwargs): - """Generate validation for the challenge.""" - raise NotImplementedError() + """Generate validation. + + :param JWK account_key: + :param unicode domain: Domain verified by the challenge. + :param OpenSSL.crypto.PKey cert_key: Optional private key used + in certificate generation. If not provided (``None``), then + fresh key will be generated. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + return self.response(account_key).gen_cert( + key=kwargs.get('cert_key'), + domain=kwargs.get('domain')) + + @staticmethod + def is_supported(): + """ + Check if TLS-ALPN-01 challenge is supported on this machine. + This implies that a recent version of OpenSSL is installed (>= 1.0.2), + or a recent cryptography version shipped with the OpenSSL library is installed. + + :returns: ``True`` if TLS-ALPN-01 is supported on this machine, ``False`` otherwise. + :rtype: bool + + """ + return (hasattr(SSL.Connection, "set_alpn_protos") + and hasattr(SSL.Context, "set_alpn_select_callback")) @Challenge.register diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index dc8fedad0..f8b7e2b30 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -27,19 +27,41 @@ logger = logging.getLogger(__name__) _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore -class SSLSocket(object): +class _DefaultCertSelection(object): + def __init__(self, certs): + self.certs = certs + + def __call__(self, connection): + server_name = connection.get_servername() + return self.certs.get(server_name, None) + + +class SSLSocket(object): # pylint: disable=too-few-public-methods """SSL wrapper for sockets. :ivar socket sock: Original wrapped socket. :ivar dict certs: Mapping from domain names (`bytes`) to `OpenSSL.crypto.X509`. :ivar method: See `OpenSSL.SSL.Context` for allowed values. + :ivar alpn_selection: Hook to select negotiated ALPN protocol for + connection. + :ivar cert_selection: Hook to select certificate for connection. If given, + `certs` parameter would be ignored, and therefore must be empty. """ - def __init__(self, sock, certs, method=_DEFAULT_SSL_METHOD): + def __init__(self, sock, certs=None, + method=_DEFAULT_SSL_METHOD, alpn_selection=None, + cert_selection=None): self.sock = sock - self.certs = certs + self.alpn_selection = alpn_selection self.method = method + if not cert_selection and not certs: + raise ValueError("Neither cert_selection or certs specified.") + if cert_selection and certs: + raise ValueError("Both cert_selection and certs specified.") + if cert_selection is None: + cert_selection = _DefaultCertSelection(certs) + self.cert_selection = cert_selection def __getattr__(self, name): return getattr(self.sock, name) @@ -56,18 +78,19 @@ class SSLSocket(object): :type connection: :class:`OpenSSL.Connection` """ - server_name = connection.get_servername() - try: - key, cert = self.certs[server_name] - except KeyError: - logger.debug("Server name (%s) not recognized, dropping SSL", - server_name) + pair = self.cert_selection(connection) + if pair is None: + logger.debug("Certificate selection for server name %s failed, dropping SSL", + connection.get_servername()) return + key, cert = pair new_context = SSL.Context(self.method) new_context.set_options(SSL.OP_NO_SSLv2) new_context.set_options(SSL.OP_NO_SSLv3) new_context.use_privatekey(key) new_context.use_certificate(cert) + if self.alpn_selection is not None: + new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) class FakeConnection(object): @@ -92,6 +115,8 @@ class SSLSocket(object): context.set_options(SSL.OP_NO_SSLv2) context.set_options(SSL.OP_NO_SSLv3) context.set_tlsext_servername_callback(self._pick_certificate_cb) + if self.alpn_selection is not None: + context.set_alpn_select_callback(self.alpn_selection) ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) ssl_sock.set_accept_state() @@ -107,8 +132,9 @@ class SSLSocket(object): return ssl_sock, addr -def probe_sni(name, host, port=443, timeout=300, - method=_DEFAULT_SSL_METHOD, source_address=('', 0)): +def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-arguments + method=_DEFAULT_SSL_METHOD, source_address=('', 0), + alpn_protocols=None): """Probe SNI server for SSL certificate. :param bytes name: Byte string to send as the server name in the @@ -120,6 +146,8 @@ def probe_sni(name, host, port=443, timeout=300, :param tuple source_address: Enables multi-path probing (selection of source interface). See `socket.creation_connection` for more info. Available only in Python 2.7+. + :param alpn_protocols: Protocols to request using ALPN. + :type alpn_protocols: `list` of `bytes` :raises acme.errors.Error: In case of any problems. @@ -149,6 +177,8 @@ def probe_sni(name, host, port=443, timeout=300, client_ssl = SSL.Connection(context, client) client_ssl.set_connect_state() client_ssl.set_tlsext_host_name(name) # pyOpenSSL>=0.13 + if alpn_protocols is not None: + client_ssl.set_alpn_protos(alpn_protocols) try: client_ssl.do_handshake() client_ssl.shutdown() @@ -239,12 +269,14 @@ def _pyopenssl_cert_or_req_san(cert_or_req): def gen_ss_cert(key, domains, not_before=None, - validity=(7 * 24 * 60 * 60), force_san=True): + validity=(7 * 24 * 60 * 60), force_san=True, extensions=None): """Generate new self-signed certificate. :type domains: `list` of `unicode` :param OpenSSL.crypto.PKey key: :param bool force_san: + :param extensions: List of additional extensions to include in the cert. + :type extensions: `list` of `OpenSSL.crypto.X509Extension` If more than one domain is provided, all of the domains are put into ``subjectAltName`` X.509 extension and first domain is set as the @@ -257,10 +289,13 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16)) cert.set_version(2) - extensions = [ + if extensions is None: + extensions = [] + + extensions.append( crypto.X509Extension( b"basicConstraints", True, b"CA:TRUE, pathlen:0"), - ] + ) cert.get_subject().CN = domains[0] # TODO: what to put into cert.get_subject()? diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 236f2c234..52ac07915 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -33,7 +33,14 @@ class TLSServer(socketserver.TCPServer): def _wrap_sock(self): self.socket = crypto_util.SSLSocket( - self.socket, certs=self.certs, method=self.method) + self.socket, cert_selection=self._cert_selection, + alpn_selection=getattr(self, '_alpn_selection', None), + method=self.method) + + def _cert_selection(self, connection): # pragma: no cover + """Callback selecting certificate for connection.""" + server_name = connection.get_servername() + return self.certs.get(server_name, None) def server_bind(self): self._wrap_sock() @@ -120,6 +127,40 @@ class BaseDualNetworkedServers(object): self.threads = [] +class TLSALPN01Server(TLSServer, ACMEServerMixin): + """TLSALPN01 Server.""" + + ACME_TLS_1_PROTOCOL = b"acme-tls/1" + + def __init__(self, server_address, certs, challenge_certs, ipv6=False): + TLSServer.__init__( + self, server_address, _BaseRequestHandlerWithLogging, certs=certs, + ipv6=ipv6) + self.challenge_certs = challenge_certs + + def _cert_selection(self, connection): + # TODO: We would like to serve challenge cert only if asked for it via + # ALPN. To do this, we need to retrieve the list of protos from client + # hello, but this is currently impossible with openssl [0], and ALPN + # negotiation is done after cert selection. + # Therefore, currently we always return challenge cert, and terminate + # handshake in alpn_selection() if ALPN protos are not what we expect. + # [0] https://github.com/openssl/openssl/issues/4952 + server_name = connection.get_servername() + logger.debug("Serving challenge cert for server name %s", server_name) + return self.challenge_certs.get(server_name, None) + + def _alpn_selection(self, _connection, alpn_protos): + """Callback to select alpn protocol.""" + if len(alpn_protos) == 1 and alpn_protos[0] == self.ACME_TLS_1_PROTOCOL: + logger.debug("Agreed on %s ALPN", self.ACME_TLS_1_PROTOCOL) + return self.ACME_TLS_1_PROTOCOL + logger.debug("Cannot agree on ALPN proto. Got: %s", str(alpn_protos)) + # Explicitly close the connection now, by returning an empty string. + # See https://www.pyopenssl.org/en/stable/api/ssl.html#OpenSSL.SSL.Context.set_alpn_select_callback # pylint: disable=line-too-long + return b"" + + class HTTPServer(BaseHTTPServer.HTTPServer): """Generic HTTP Server.""" @@ -222,3 +263,16 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """ return functools.partial( cls, simple_http_resources=simple_http_resources) + + +class _BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler): + """BaseRequestHandler with logging.""" + + def log_message(self, format, *args): # pylint: disable=redefined-builtin + """Log arbitrary message.""" + logger.debug("%s - - %s", self.client_address[0], format % args) + + def handle(self): + """Handle request.""" + self.log_message("Incoming request") + socketserver.BaseRequestHandler.handle(self) diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index adebaffc5..2b44d677d 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -2,10 +2,13 @@ import unittest import josepy as jose +import OpenSSL import mock import requests from six.moves.urllib import parse as urllib_parse +from acme import errors + import test_util CERT = test_util.load_comparable_cert('cert.pem') @@ -256,30 +259,87 @@ class HTTP01Test(unittest.TestCase): class TLSALPN01ResponseTest(unittest.TestCase): def setUp(self): - from acme.challenges import TLSALPN01Response - self.msg = TLSALPN01Response(key_authorization=u'foo') + from acme.challenges import TLSALPN01 + self.chall = TLSALPN01( + token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e')) + self.domain = u'example.com' + self.domain2 = u'example2.com' + + self.response = self.chall.response(KEY) self.jmsg = { 'resource': 'challenge', 'type': 'tls-alpn-01', - 'keyAuthorization': u'foo', + 'keyAuthorization': self.response.key_authorization, } - from acme.challenges import TLSALPN01 - self.chall = TLSALPN01(token=(b'x' * 16)) - self.response = self.chall.response(KEY) - def test_to_partial_json(self): self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, - self.msg.to_partial_json()) + self.response.to_partial_json()) def test_from_json(self): from acme.challenges import TLSALPN01Response - self.assertEqual(self.msg, TLSALPN01Response.from_json(self.jmsg)) + self.assertEqual(self.response, TLSALPN01Response.from_json(self.jmsg)) def test_from_json_hashable(self): from acme.challenges import TLSALPN01Response hash(TLSALPN01Response.from_json(self.jmsg)) + def test_gen_verify_cert(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_gen_verify_cert_gen_key(self): + cert, key = self.response.gen_cert(self.domain) + self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_verify_bad_cert(self): + self.assertFalse(self.response.verify_cert(self.domain, + test_util.load_cert('cert.pem'))) + + def test_verify_bad_domain(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertFalse(self.response.verify_cert(self.domain2, cert)) + + def test_simple_verify_bad_key_authorization(self): + key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) + self.response.simple_verify(self.chall, "local", key2.public_key()) + + @mock.patch('acme.challenges.TLSALPN01Response.verify_cert', autospec=True) + def test_simple_verify(self, mock_verify_cert): + mock_verify_cert.return_value = mock.sentinel.verification + self.assertEqual( + mock.sentinel.verification, self.response.simple_verify( + self.chall, self.domain, KEY.public_key(), + cert=mock.sentinel.cert)) + mock_verify_cert.assert_called_once_with( + self.response, self.domain, mock.sentinel.cert) + + @mock.patch('acme.challenges.socket.gethostbyname') + @mock.patch('acme.challenges.crypto_util.probe_sni') + def test_probe_cert(self, mock_probe_sni, mock_gethostbyname): + mock_gethostbyname.return_value = '127.0.0.1' + self.response.probe_cert('foo.com') + mock_gethostbyname.assert_called_once_with('foo.com') + mock_probe_sni.assert_called_once_with( + host='127.0.0.1', port=self.response.PORT, name='foo.com', + alpn_protocols=['acme-tls/1']) + + self.response.probe_cert('foo.com', host='8.8.8.8') + mock_probe_sni.assert_called_with( + host='8.8.8.8', port=mock.ANY, name='foo.com', + alpn_protocols=['acme-tls/1']) + + @mock.patch('acme.challenges.TLSALPN01Response.probe_cert') + def test_simple_verify_false_on_probe_error(self, mock_probe_cert): + mock_probe_cert.side_effect = errors.Error + self.assertFalse(self.response.simple_verify( + self.chall, self.domain, KEY.public_key())) + class TLSALPN01Test(unittest.TestCase): @@ -309,8 +369,13 @@ class TLSALPN01Test(unittest.TestCase): self.assertRaises( jose.DeserializationError, TLSALPN01.from_json, self.jmsg) - def test_validation(self): - self.assertRaises(NotImplementedError, self.msg.validation, KEY) + @mock.patch('acme.challenges.TLSALPN01Response.gen_cert') + def test_validation(self, mock_gen_cert): + mock_gen_cert.return_value = ('cert', 'key') + self.assertEqual(('cert', 'key'), self.msg.validation( + KEY, cert_key=mock.sentinel.cert_key, domain=mock.sentinel.domain)) + mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key, + domain=mock.sentinel.domain) class DNSTest(unittest.TestCase): diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index 41640ed60..ff08a5405 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -18,7 +18,6 @@ import test_util class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" - def setUp(self): self.cert = test_util.load_comparable_cert('rsa2048_cert.pem') key = test_util.load_pyopenssl_private_key('rsa2048_key.pem') @@ -32,7 +31,8 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init def server_bind(self): # pylint: disable=missing-docstring - self.socket = SSLSocket(socket.socket(), certs=certs) + self.socket = SSLSocket(socket.socket(), + certs) socketserver.TCPServer.server_bind(self) self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) @@ -73,6 +73,18 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): socket.setdefaulttimeout(original_timeout) +class SSLSocketTest(unittest.TestCase): + """Tests for acme.crypto_util.SSLSocket.""" + + def test_ssl_socket_invalid_arguments(self): + from acme.crypto_util import SSLSocket + with self.assertRaises(ValueError): + _ = SSLSocket(None, {'sni': ('key', 'cert')}, + cert_selection=lambda _: None) + with self.assertRaises(ValueError): + _ = SSLSocket(None) + + class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 83ced12b0..e2817b29c 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -10,7 +10,10 @@ from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import challenges +from acme import crypto_util +from acme import errors from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module + import test_util @@ -84,6 +87,59 @@ class HTTP01ServerTest(unittest.TestCase): self.assertFalse(self._test_http01(add=False)) +@unittest.skipIf(not challenges.TLSALPN01.is_supported(), "pyOpenSSL too old") +class TLSALPN01ServerTest(unittest.TestCase): + """Test for acme.standalone.TLSALPN01Server.""" + + def setUp(self): + self.certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa2048_key.pem'), + test_util.load_cert('rsa2048_cert.pem'), + )} + # Use different certificate for challenge. + self.challenge_certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa1024_key.pem'), + test_util.load_cert('rsa1024_cert.pem'), + )} + from acme.standalone import TLSALPN01Server + self.server = TLSALPN01Server(("localhost", 0), certs=self.certs, + challenge_certs=self.challenge_certs) + # pylint: disable=no-member + self.thread = threading.Thread(target=self.server.serve_forever) + self.thread.start() + + def tearDown(self): + self.server.shutdown() # pylint: disable=no-member + self.thread.join() + + # TODO: This is not implemented yet, see comments in standalone.py + # def test_certs(self): + # host, port = self.server.socket.getsockname()[:2] + # cert = crypto_util.probe_sni( + # b'localhost', host=host, port=port, timeout=1) + # # Expect normal cert when connecting without ALPN. + # self.assertEqual(jose.ComparableX509(cert), + # jose.ComparableX509(self.certs[b'localhost'][1])) + + def test_challenge_certs(self): + host, port = self.server.socket.getsockname()[:2] + cert = crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"acme-tls/1"]) + # Expect challenge cert when connecting with ALPN. + self.assertEqual( + jose.ComparableX509(cert), + jose.ComparableX509(self.challenge_certs[b'localhost'][1]) + ) + + def test_bad_alpn(self): + host, port = self.server.socket.getsockname()[:2] + with self.assertRaises(errors.Error): + crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"bad-alpn"]) + + class BaseDualNetworkedServersTest(unittest.TestCase): """Test for acme.standalone.BaseDualNetworkedServers.""" @@ -138,7 +194,6 @@ class BaseDualNetworkedServersTest(unittest.TestCase): class HTTP01DualNetworkedServersTest(unittest.TestCase): """Tests for acme.standalone.HTTP01DualNetworkedServers.""" - def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) diff --git a/acme/tests/testdata/README b/acme/tests/testdata/README index dfe3f5405..d65cc3018 100644 --- a/acme/tests/testdata/README +++ b/acme/tests/testdata/README @@ -10,6 +10,8 @@ and for the CSR: openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -outform DER > csr.der -and for the certificate: +and for the certificates: - openssl req -key rsa2047_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 > rsa2048_cert.pem + openssl req -key rsa1024_key.pem -new -subj '/CN=example.com' -x509 > rsa1024_cert.pem diff --git a/acme/tests/testdata/rsa1024_cert.pem b/acme/tests/testdata/rsa1024_cert.pem new file mode 100644 index 000000000..1b7912181 --- /dev/null +++ b/acme/tests/testdata/rsa1024_cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB/TCCAWagAwIBAgIJAOyRIBs3QT8QMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV +BAMMC2V4YW1wbGUuY29tMB4XDTE4MDQyMzEwMzE0NFoXDTE4MDUyMzEwMzE0NFow +FjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ +AoGBAJqJ87R8aVwByONxgQA9hwgvQd/QqI1r1UInXhEF2VnEtZGtUWLi100IpIqr +Mq4qusDwNZ3g8cUPtSkvJGs89djoajMDIJP7lQUEKUYnYrI0q755Tr/DgLWSk7iW +l5ezym0VzWUD0/xXUz8yRbNMTjTac80rS5SZk2ja2wWkYlRJAgMBAAGjUzBRMB0G +A1UdDgQWBBSsaX0IVZ4XXwdeffVAbG7gnxSYjTAfBgNVHSMEGDAWgBSsaX0IVZ4X +XwdeffVAbG7gnxSYjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB +ADe7SVmvGH2nkwVfONk8TauRUDkePN1CJZKFb2zW1uO9ANJ2v5Arm/OQp0BG/xnI +Djw/aLTNVESF89oe15dkrUErtcaF413MC1Ld5lTCaJLHLGqDKY69e02YwRuxW7jY +qarpt7k7aR5FbcfO5r4V/FK/Gvp4Dmoky8uap7SJIW6x +-----END CERTIFICATE----- diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 2f22b5204..36547fdd1 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -11,6 +11,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). the `manual` plugin: `CERTBOT_REMAINING_CHALLENGES` is equal to the number of challenges remaining after the current challenge, `CERTBOT_ALL_DOMAINS` is a comma-separated list of all domains challenged for the current certificate. +* Added TLS-ALPN-01 challenge support in the `acme` library. Support of this + challenge in the Certbot client is planned to be added in a future release. ### Changed From 809cb516c918575bc1688141dfe9b4da001d6570 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 13 Mar 2020 17:56:35 +0100 Subject: [PATCH 58/92] Fix acme compliance to RFC 8555 (#7176) This PR is an alternative to #7125. Instead of disabling the strict mode on Pebble, this PR fixes the JWS payloads regarding RFC 8555 to be compliant, and allow certbot to work with Pebble v2.1.0+. * Fix acme compliance to RFC 8555. * Working mixin * Activate back pebble strict mode * Use mixin for type * Update dependencies * Fix also in fields_to_partial_json * Update pebble * Add changelog --- acme/acme/challenges.py | 3 +- acme/acme/client.py | 3 + acme/acme/messages.py | 13 ++-- acme/acme/mixins.py | 65 +++++++++++++++++++ acme/tests/challenges_test.py | 13 ++++ acme/tests/client_test.py | 3 +- acme/tests/messages_test.py | 14 ++++ .../utils/acme_server.py | 2 +- .../utils/pebble_artifacts.py | 2 +- certbot-nginx/local-oldest-requirements.txt | 4 +- certbot-nginx/setup.py | 4 +- certbot/CHANGELOG.md | 3 +- certbot/local-oldest-requirements.txt | 2 +- certbot/setup.py | 2 +- 14 files changed, 116 insertions(+), 17 deletions(-) create mode 100644 acme/acme/mixins.py diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 0b112be00..b9c6b7eb2 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -16,6 +16,7 @@ from OpenSSL import crypto from acme import crypto_util from acme import errors from acme import fields +from acme.mixins import ResourceMixin, TypeMixin logger = logging.getLogger(__name__) @@ -34,7 +35,7 @@ class Challenge(jose.TypedJSONObjectWithFields): return UnrecognizedChallenge.from_json(jobj) -class ChallengeResponse(jose.TypedJSONObjectWithFields): +class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge response.""" TYPES = {} # type: dict diff --git a/acme/acme/client.py b/acme/acme/client.py index cecb727c7..cbe543f91 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -25,6 +25,7 @@ from acme.magic_typing import Dict from acme.magic_typing import List from acme.magic_typing import Set from acme.magic_typing import Text +from acme.mixins import VersionedLEACMEMixin logger = logging.getLogger(__name__) @@ -987,6 +988,8 @@ class ClientNetwork(object): :rtype: `josepy.JWS` """ + if isinstance(obj, VersionedLEACMEMixin): + obj.le_acme_version = acme_version jobj = obj.json_dumps(indent=2).encode() if obj else b'' logger.debug('JWS payload:\n%s', jobj) kwargs = { diff --git a/acme/acme/messages.py b/acme/acme/messages.py index f8f4bfbe7..90059a6fb 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -9,6 +9,7 @@ from acme import errors from acme import fields from acme import jws from acme import util +from acme.mixins import ResourceMixin try: from collections.abc import Hashable @@ -356,13 +357,13 @@ class Registration(ResourceBody): @Directory.register -class NewRegistration(Registration): +class NewRegistration(ResourceMixin, Registration): """New registration.""" resource_type = 'new-reg' resource = fields.Resource(resource_type) -class UpdateRegistration(Registration): +class UpdateRegistration(ResourceMixin, Registration): """Update registration.""" resource_type = 'reg' resource = fields.Resource(resource_type) @@ -498,13 +499,13 @@ class Authorization(ResourceBody): @Directory.register -class NewAuthorization(Authorization): +class NewAuthorization(ResourceMixin, Authorization): """New authorization.""" resource_type = 'new-authz' resource = fields.Resource(resource_type) -class UpdateAuthorization(Authorization): +class UpdateAuthorization(ResourceMixin, Authorization): """Update authorization.""" resource_type = 'authz' resource = fields.Resource(resource_type) @@ -522,7 +523,7 @@ class AuthorizationResource(ResourceWithURI): @Directory.register -class CertificateRequest(jose.JSONObjectWithFields): +class CertificateRequest(ResourceMixin, jose.JSONObjectWithFields): """ACME new-cert request. :ivar josepy.util.ComparableX509 csr: @@ -548,7 +549,7 @@ class CertificateResource(ResourceWithURI): @Directory.register -class Revocation(jose.JSONObjectWithFields): +class Revocation(ResourceMixin, jose.JSONObjectWithFields): """Revocation message. :ivar .ComparableX509 certificate: `OpenSSL.crypto.X509` wrapped in diff --git a/acme/acme/mixins.py b/acme/acme/mixins.py new file mode 100644 index 000000000..1cd050ccc --- /dev/null +++ b/acme/acme/mixins.py @@ -0,0 +1,65 @@ +"""Useful mixins for Challenge and Resource objects""" + + +class VersionedLEACMEMixin(object): + """This mixin stores the version of Let's Encrypt's endpoint being used.""" + @property + def le_acme_version(self): + """Define the version of ACME protocol to use""" + return getattr(self, '_le_acme_version', 1) + + @le_acme_version.setter + def le_acme_version(self, version): + # We need to use object.__setattr__ to not depend on the specific implementation of + # __setattr__ in current class (eg. jose.TypedJSONObjectWithFields raises AttributeError + # for any attempt to set an attribute to make objects immutable). + object.__setattr__(self, '_le_acme_version', version) + + def __setattr__(self, key, value): + if key == 'le_acme_version': + # Required for @property to operate properly. See comment above. + object.__setattr__(self, key, value) + else: + super(VersionedLEACMEMixin, self).__setattr__(key, value) # pragma: no cover + + +class ResourceMixin(VersionedLEACMEMixin): + """ + This mixin generates a RFC8555 compliant JWS payload + by removing the `resource` field if needed (eg. ACME v2 protocol). + """ + def to_partial_json(self): + """See josepy.JSONDeserializable.to_partial_json()""" + return _safe_jobj_compliance(super(ResourceMixin, self), + 'to_partial_json', 'resource') + + def fields_to_partial_json(self): + """See josepy.JSONObjectWithFields.fields_to_partial_json()""" + return _safe_jobj_compliance(super(ResourceMixin, self), + 'fields_to_partial_json', 'resource') + + +class TypeMixin(VersionedLEACMEMixin): + """ + This mixin allows generation of a RFC8555 compliant JWS payload + by removing the `type` field if needed (eg. ACME v2 protocol). + """ + def to_partial_json(self): + """See josepy.JSONDeserializable.to_partial_json()""" + return _safe_jobj_compliance(super(TypeMixin, self), + 'to_partial_json', 'type') + + def fields_to_partial_json(self): + """See josepy.JSONObjectWithFields.fields_to_partial_json()""" + return _safe_jobj_compliance(super(TypeMixin, self), + 'fields_to_partial_json', 'type') + + +def _safe_jobj_compliance(instance, jobj_method, uncompliant_field): + if hasattr(instance, jobj_method): + jobj = getattr(instance, jobj_method)() + if instance.le_acme_version == 2: + jobj.pop(uncompliant_field, None) + return jobj + + raise AttributeError('Method {0}() is not implemented.'.format(jobj_method)) # pragma: no cover diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index 2b44d677d..433c7bb82 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -478,5 +478,18 @@ class DNSResponseTest(unittest.TestCase): self.msg.check_validation(self.chall, KEY.public_key())) +class JWSPayloadRFC8555Compliant(unittest.TestCase): + """Test for RFC8555 compliance of JWS generated from resources/challenges""" + def test_challenge_payload(self): + from acme.challenges import HTTP01Response + + challenge_body = HTTP01Response() + challenge_body.le_acme_version = 2 + + jobj = challenge_body.json_dumps(indent=2).encode() + # RFC8555 states that challenge responses must have an empty payload. + self.assertEqual(jobj, b'{}') + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index a4966140f..1e132d79f 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -16,6 +16,7 @@ from acme import errors from acme import jws as acme_jws from acme import messages from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module +from acme.mixins import VersionedLEACMEMixin import messages_test import test_util @@ -886,7 +887,7 @@ class ClientV2Test(ClientTestBase): self.client.net.get.assert_not_called() -class MockJSONDeSerializable(jose.JSONDeSerializable): +class MockJSONDeSerializable(VersionedLEACMEMixin, jose.JSONDeSerializable): # pylint: disable=missing-docstring def __init__(self, value): self.value = value diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index b9b70266b..d53fb764c 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -453,6 +453,7 @@ class OrderResourceTest(unittest.TestCase): 'authorizations': None, }) + class NewOrderTest(unittest.TestCase): """Tests for acme.messages.NewOrder.""" @@ -467,5 +468,18 @@ class NewOrderTest(unittest.TestCase): }) +class JWSPayloadRFC8555Compliant(unittest.TestCase): + """Test for RFC8555 compliance of JWS generated from resources/challenges""" + def test_message_payload(self): + from acme.messages import NewAuthorization + + new_order = NewAuthorization() + new_order.le_acme_version = 2 + + jobj = new_order.json_dumps(indent=2).encode() + # RFC8555 states that JWS bodies must not have a resource field. + self.assertEqual(jobj, b'{}') + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 5483251e6..b14bd4a32 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -131,7 +131,7 @@ class ACMEServer(object): environ['PEBBLE_AUTHZREUSE'] = '100' self._launch_process( - [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053'], + [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053', '-strict'], env=environ) self._launch_process( diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index 2b1557928..7fe03b990 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -7,7 +7,7 @@ import requests from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT -PEBBLE_VERSION = 'v2.2.1' +PEBBLE_VERSION = 'v2.3.0' ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index cee142934..1782f15ba 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,3 +1,3 @@ # Remember to update setup.py to match the package versions below. -acme[dev]==1.0.0 -certbot[dev]==1.1.0 +-e acme[dev] +-e certbot[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 34785f963..0d62e7d55 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -9,8 +9,8 @@ version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=1.0.0', - 'certbot>=1.1.0', + 'acme>=1.4.0.dev0', + 'certbot>=1.4.0.dev0', 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 36547fdd1..0bbc9b7ff 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -20,7 +20,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* +* When using an RFC 8555 compliant endpoint, the `acme` library no longer sends the + `resource` field in any requests or the `type` field when responding to challenges. More details about these changes can be found on our GitHub repo. diff --git a/certbot/local-oldest-requirements.txt b/certbot/local-oldest-requirements.txt index f6d158890..0acc68652 100644 --- a/certbot/local-oldest-requirements.txt +++ b/certbot/local-oldest-requirements.txt @@ -1,2 +1,2 @@ # Remember to update setup.py to match the package versions below. -acme[dev]==0.40.0 +-e acme[dev] diff --git a/certbot/setup.py b/certbot/setup.py index d19327e5e..514aec8c5 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -36,7 +36,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=0.40.0', + 'acme>=1.4.0.dev0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. From 9a256ca4fee624a5aa1b901de939f0dc6dd4e641 Mon Sep 17 00:00:00 2001 From: Blake Bourque Date: Fri, 13 Mar 2020 15:26:15 -0400 Subject: [PATCH 59/92] Fix plugin links --- certbot/docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 25d832761..06d0e1b8d 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -247,8 +247,8 @@ built-in Standalone authenticator, implement just one interface. There are also `~certbot.interfaces.IDisplay` plugins, which can change how prompts are displayed to a user. -.. _interfaces.py: https://github.com/certbot/certbot/blob/master/certbot/interfaces.py -.. _plugins/common.py: https://github.com/certbot/certbot/blob/master/certbot/plugins/common.py#L34 +.. _interfaces.py: https://github.com/certbot/certbot/blob/master/certbot/certbot/interfaces.py +.. _plugins/common.py: https://github.com/certbot/certbot/blob/master/certbot/certbot/plugins/common.py#L45 Authenticators From 06599a1e18cd809dc4db26990da67fcdfc3316d4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 16 Mar 2020 09:43:48 -0700 Subject: [PATCH 60/92] Cleanup more pylint issues (#7848) This PR builds on #7657 and cleans up additional unnecessary pylint comments and some stray comments referring to pylint: disable comments that have been deleted that I didn't notice in my review of that PR. * Remove stray pylint link. * Cleanup more pylint comments * Cleanup magic_typing imports * Remove unneeded pylint: enable comments --- acme/acme/magic_typing.py | 1 - acme/tests/client_test.py | 1 - acme/tests/crypto_util_test.py | 1 - acme/tests/magic_typing_test.py | 4 ++-- acme/tests/messages_test.py | 1 - acme/tests/standalone_test.py | 1 - certbot-apache/certbot_apache/_internal/augeasparser.py | 2 +- certbot-apache/certbot_apache/_internal/entrypoint.py | 2 -- certbot-apache/certbot_apache/_internal/interfaces.py | 1 - certbot-apache/tests/augeasnode_test.py | 1 - certbot-apache/tests/http_01_test.py | 1 - certbot-nginx/certbot_nginx/_internal/configurator.py | 1 - certbot-nginx/tests/parser_test.py | 1 - certbot/certbot/_internal/cli/helpful.py | 4 +--- certbot/certbot/_internal/plugins/disco.py | 3 --- certbot/certbot/compat/filesystem.py | 3 --- certbot/tests/compat/filesystem_test.py | 2 -- certbot/tests/display/completer_test.py | 1 - certbot/tests/error_handler_test.py | 3 --- certbot/tests/hook_test.py | 1 - certbot/tests/log_test.py | 1 - certbot/tests/main_test.py | 1 - certbot/tests/plugins/disco_test.py | 1 - certbot/tests/plugins/selection_test.py | 1 - certbot/tests/plugins/standalone_test.py | 3 --- 25 files changed, 4 insertions(+), 38 deletions(-) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index d6b1ff056..7c5231c75 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -11,6 +11,5 @@ try: # mypy doesn't respect modifying sys.modules from typing import * # pylint: disable=wildcard-import, unused-wildcard-import from typing import Collection, IO # type: ignore - # pylint: enable=unused-import except ImportError: sys.modules[__name__] = TypingClass() diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 1e132d79f..010974a32 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -15,7 +15,6 @@ from acme import challenges from acme import errors from acme import jws as acme_jws from acme import messages -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from acme.mixins import VersionedLEACMEMixin import messages_test import test_util diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index ff08a5405..705a3c856 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -11,7 +11,6 @@ import six from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import errors -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module import test_util diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 23dfe3367..60b4a5df4 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -18,7 +18,7 @@ class MagicTypingTest(unittest.TestCase): sys.modules['typing'] = typing_class_mock if 'acme.magic_typing' in sys.modules: del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text # pylint: disable=no-name-in-module + from acme.magic_typing import Text self.assertEqual(Text, text_mock) del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing @@ -31,7 +31,7 @@ class MagicTypingTest(unittest.TestCase): sys.modules['typing'] = None if 'acme.magic_typing' in sys.modules: del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text # pylint: disable=no-name-in-module + from acme.magic_typing import Text self.assertTrue(Text is None) del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index d53fb764c..d36e2cc99 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -5,7 +5,6 @@ import josepy as jose import mock from acme import challenges -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module import test_util CERT = test_util.load_comparable_cert('cert.der') diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index e2817b29c..8c08ab89b 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -12,7 +12,6 @@ from six.moves import socketserver # type: ignore # pylint: disable=import-err from acme import challenges from acme import crypto_util from acme import errors -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module import test_util diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index e1d7c941d..f85d80923 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -64,7 +64,7 @@ Translates over to: "/files/etc/apache2/apache2.conf/bLoCk[1]", ] """ -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Set from certbot import errors from certbot.compat import os diff --git a/certbot-apache/certbot_apache/_internal/entrypoint.py b/certbot-apache/certbot_apache/_internal/entrypoint.py index e31e1f4eb..79337b381 100644 --- a/certbot-apache/certbot_apache/_internal/entrypoint.py +++ b/certbot-apache/certbot_apache/_internal/entrypoint.py @@ -1,6 +1,4 @@ """ Entry point for Apache Plugin """ -# Pylint does not like disutils.version when running inside a venv. -# See: https://github.com/PyCQA/pylint/issues/73 from distutils.version import LooseVersion from certbot import util diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 1b67be5c8..647790c41 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -102,7 +102,6 @@ For this reason the internal representation of data should not ignore the case. import abc import six -from acme.magic_typing import Any, Dict, Optional, Tuple # pylint: disable=unused-import, no-name-in-module @six.add_metaclass(abc.ABCMeta) diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 9d663a05f..8417bc283 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -3,7 +3,6 @@ import mock import util -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot_apache._internal import assertions diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 422a76443..85b17ca28 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -5,7 +5,6 @@ import errno import mock from acme import challenges -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot.compat import filesystem diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 459950aa1..ddab48512 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -1,5 +1,4 @@ """Nginx Configuration""" -# https://github.com/PyCQA/pylint/issues/73 from distutils.version import LooseVersion import logging import re diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index fd5d338d3..72cfc0716 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -4,7 +4,6 @@ import re import shutil import unittest -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot.compat import os from certbot_nginx._internal import nginxparser diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index e63ab4b87..31d9396e5 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -11,9 +11,7 @@ import zope.interface from zope.interface import interfaces as zope_interfaces -# pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Any, Dict, Optional -# pylint: enable=unused-import, no-name-in-module +from acme.magic_typing import Any, Dict from certbot import crypto_util from certbot import errors diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index d98a4cb0c..f1d89f06a 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -192,9 +192,6 @@ class PluginsRegistry(Mapping): # This prevents deadlock caused by plugins acquiring a lock # and ensures at least one concurrent Certbot instance will run # successfully. - - # Pylint checks for super init, but also claims the super - # has no __init__member self._plugins = collections.OrderedDict(sorted(six.iteritems(plugins))) @classmethod diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index 88c2916fa..b9b6e5cc6 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -6,8 +6,6 @@ import os # pylint: disable=os-module-forbidden import stat from acme.magic_typing import List -from acme.magic_typing import Tuple # pylint: disable=unused-import -from acme.magic_typing import Union # pylint: disable=unused-import try: import ntsecuritycon @@ -17,7 +15,6 @@ try: import win32file import pywintypes import winerror - # pylint: enable=import-error except ImportError: POSIX_MODE = True else: diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index e721bbd48..fdfb1ffe9 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -13,11 +13,9 @@ import certbot.tests.util as test_util from certbot.tests.util import TempDirTestCase try: - # pylint: disable=import-error import win32api import win32security import ntsecuritycon - # pylint: enable=import-error POSIX_MODE = False except ImportError: POSIX_MODE = True diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 5ddf69266..a183fd14f 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -10,7 +10,6 @@ import unittest import mock from six.moves import reload_module # pylint: disable=import-error -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.compat import filesystem # pylint: disable=ungrouped-imports from certbot.compat import os # pylint: disable=ungrouped-imports import certbot.tests.util as test_util # pylint: disable=ungrouped-imports diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 45fec7f39..011313208 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -6,9 +6,6 @@ import unittest import mock -from acme.magic_typing import Callable # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module from certbot.compat import os diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index a3bba57d2..54d0fcf67 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -3,7 +3,6 @@ import unittest import mock -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import util from certbot.compat import filesystem diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 3b9adbbf2..5b0918ce5 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -9,7 +9,6 @@ import mock import six from acme import messages -from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import util from certbot._internal import constants diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 7ebe5e66a..8b2645876 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -18,7 +18,6 @@ import pytz import six from six.moves import reload_module # pylint: disable=import-error -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import crypto_util from certbot import errors from certbot import interfaces # pylint: disable=unused-import diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index 6d3c7d97e..eec0795e3 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -8,7 +8,6 @@ import pkg_resources import six import zope.interface -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot._internal.plugins import standalone diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index ac846af7b..c66473ad1 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -5,7 +5,6 @@ import unittest import mock import zope.component -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot._internal.plugins.disco import PluginsRegistry diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 5d9ff5244..701abe109 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -11,9 +11,6 @@ import six from acme import challenges from acme import standalone as acme_standalone # pylint: disable=unused-import -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot.tests import acme_util From 9e3c348dff75e6db4c27ae3e343d9e151d50862e Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 23 Mar 2020 16:49:52 -0700 Subject: [PATCH 61/92] Disable TLS session tickets in Apache (#7771) Fixes #7350. This PR changes the parsed modules from a `set` to a `dict`, with the filepath argument as the value. Accordingly, after calling `enable_mod` to enable `ssl_module`, modules now need to be re-parsed, so call `reset_modules`. * Add mechanism for selecting apache config file, based on work done in #7191. * Check OpenSSL version * Remove os imports * debian override still needs os * Reformat remaining apache tests with modules dict syntax * Clean up more apache tests * Switch from property to method for openssl and add tests for coverage. * Sometimes the dict location will be None in which case we should in fact return None * warn thoroughly and consistently in openssl_version function * update tests for new warnings * read file as bytes, and factor out the open for testing * normalize ssl_module_location path to account for being relative to server root * Use byte literals in a python 2 and 3 compatible way * string does need to be a literal * patch builtins open * add debug, remove space * Add test to check if OpenSSL detection is working on different systems * fix relative test location for cwd * put on its own line in test case * Revert test file to status in master. * Call augeas load before reparsing modules to pick up the changes * fix grep, tail, and mod_ssl location on centos * strip the trailing whitespace from fedora * just use LooseVersion in test * call apache2ctl on debian systems * Use sudo for apache2ctl command * add check to make sure we're getting a version * Add boolean so we don't warn on debian/ubuntu before trying to enable mod_ssl * Reduce warnings while testing by setting mock _openssl_version. * Make sure we're not throwing away any unwritten changes to the config * test last warning case for coverage * text changes for clarity --- certbot-apache/MANIFEST.in | 2 +- .../certbot_apache/_internal/apache_util.py | 13 ++ .../certbot_apache/_internal/configurator.py | 96 ++++++++- .../certbot_apache/_internal/constants.py | 1 + .../certbot_apache/_internal/override_arch.py | 4 - .../_internal/override_centos.py | 4 - .../_internal/override_darwin.py | 4 - .../_internal/override_debian.py | 3 - .../_internal/override_fedora.py | 5 - .../_internal/override_gentoo.py | 4 - .../certbot_apache/_internal/override_suse.py | 4 - .../certbot_apache/_internal/parser.py | 47 +++-- .../current-options-ssl-apache.conf | 19 ++ .../old-options-ssl-apache.conf} | 0 certbot-apache/tests/autohsts_test.py | 12 +- certbot-apache/tests/centos_test.py | 2 +- certbot-apache/tests/configurator_test.py | 193 +++++++++++++----- certbot-apache/tests/debian_test.py | 20 +- certbot-apache/tests/fedora_test.py | 2 +- certbot-apache/tests/gentoo_test.py | 2 +- certbot-apache/tests/http_01_test.py | 16 +- certbot-apache/tests/parser_test.py | 8 +- certbot-apache/tests/util.py | 6 +- certbot/CHANGELOG.md | 7 +- tests/letstest/scripts/test_apache2.sh | 16 ++ .../letstest/scripts/test_openssl_version.py | 30 +++ 26 files changed, 373 insertions(+), 147 deletions(-) create mode 100644 certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf rename certbot-apache/certbot_apache/_internal/{options-ssl-apache.conf => tls_configs/old-options-ssl-apache.conf} (100%) create mode 100644 tests/letstest/scripts/test_openssl_version.py diff --git a/certbot-apache/MANIFEST.in b/certbot-apache/MANIFEST.in index 2316983bb..9e4913a03 100644 --- a/certbot-apache/MANIFEST.in +++ b/certbot-apache/MANIFEST.in @@ -1,7 +1,7 @@ include LICENSE.txt include README.rst recursive-include tests * -include certbot_apache/_internal/options-ssl-apache.conf recursive-include certbot_apache/_internal/augeas_lens *.aug +recursive-include certbot_apache/_internal/tls_configs *.conf global-exclude __pycache__ global-exclude *.py[cod] diff --git a/certbot-apache/certbot_apache/_internal/apache_util.py b/certbot-apache/certbot_apache/_internal/apache_util.py index 085ccddc8..ebc8a26c9 100644 --- a/certbot-apache/certbot_apache/_internal/apache_util.py +++ b/certbot-apache/certbot_apache/_internal/apache_util.py @@ -5,6 +5,8 @@ import logging import re import subprocess +import pkg_resources + from certbot import errors from certbot import util @@ -241,3 +243,14 @@ def _get_runtime_cfg(command): "loaded because Apache is misconfigured.") return stdout + +def find_ssl_apache_conf(prefix): + """ + Find a TLS Apache config file in the dedicated storage. + :param str prefix: prefix of the TLS Apache config file to find + :return: the path the TLS Apache config file + :rtype: str + """ + return pkg_resources.resource_filename( + "certbot_apache", + os.path.join("_internal", "tls_configs", "{0}-options-ssl-apache.conf".format(prefix))) diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 8daa28173..fc386404d 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -1,6 +1,7 @@ """Apache Configurator.""" # pylint: disable=too-many-lines from collections import defaultdict +from distutils.version import LooseVersion import copy import fnmatch import logging @@ -8,7 +9,6 @@ import re import socket import time -import pkg_resources import six import zope.component import zope.interface @@ -110,14 +110,29 @@ class ApacheConfigurator(common.Installer): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def option(self, key): """Get a value from options""" return self.options.get(key) + def pick_apache_config(self, warn_on_no_mod_ssl=True): + """ + Pick the appropriate TLS Apache configuration file for current version of Apache and OS. + + :param bool warn_on_no_mod_ssl: True if we should warn if mod_ssl is not found. + + :return: the path to the TLS Apache configuration file to use + :rtype: str + """ + # Disabling TLS session tickets is supported by Apache 2.4.11+ and OpenSSL 1.0.2l+. + # So for old versions of Apache we pick a configuration without this option. + openssl_version = self.openssl_version(warn_on_no_mod_ssl) + if self.version < (2, 4, 11) or not openssl_version or\ + LooseVersion(openssl_version) < LooseVersion('1.0.2l'): + return apache_util.find_ssl_apache_conf("old") + return apache_util.find_ssl_apache_conf("current") + def _prepare_options(self): """ Set the values possibly changed by command line parameters to @@ -184,6 +199,7 @@ class ApacheConfigurator(common.Installer): """ version = kwargs.pop("version", None) use_parsernode = kwargs.pop("use_parsernode", False) + openssl_version = kwargs.pop("openssl_version", None) super(ApacheConfigurator, self).__init__(*args, **kwargs) # Add name_server association dict @@ -209,6 +225,7 @@ class ApacheConfigurator(common.Installer): self.parser = None self.parser_root = None self.version = version + self._openssl_version = openssl_version self.vhosts = None self.options = copy.deepcopy(self.OS_DEFAULTS) self._enhance_func = {"redirect": self._enable_redirect, @@ -225,6 +242,52 @@ class ApacheConfigurator(common.Installer): """Full absolute path to digest of updated SSL configuration file.""" return os.path.join(self.config.config_dir, constants.UPDATED_MOD_SSL_CONF_DIGEST) + def _open_module_file(self, ssl_module_location): + """Extract the open lines of openssl_version for testing purposes""" + try: + with open(ssl_module_location, mode="rb") as f: + contents = f.read() + except IOError as error: + logger.debug(str(error), exc_info=True) + return None + return contents + + def openssl_version(self, warn_on_no_mod_ssl=True): + """Lazily retrieve openssl version + + :param bool warn_on_no_mod_ssl: `True` if we should warn if mod_ssl is not found. Set to + `False` when we know we'll try to enable mod_ssl later. This is currently debian/ubuntu, + when called from `prepare`. + + :return: the OpenSSL version as a string, or None. + :rtype: str or None + """ + if self._openssl_version: + return self._openssl_version + # Step 1. Check for LoadModule directive + try: + ssl_module_location = self.parser.modules['ssl_module'] + except KeyError: + if warn_on_no_mod_ssl: + logger.warning("Could not find ssl_module; not disabling session tickets.") + return None + if not ssl_module_location: + logger.warning("Could not find ssl_module; not disabling session tickets.") + return None + ssl_module_location = self.parser.standard_path_from_server_root(ssl_module_location) + # Step 2. Grep in the .so for openssl version + contents = self._open_module_file(ssl_module_location) + if not contents: + logger.warning("Unable to read ssl_module file; not disabling session tickets.") + return None + # looks like: OpenSSL 1.0.2s 28 May 2019 + matches = re.findall(br"OpenSSL ([0-9]\.[^ ]+) ", contents) + if not matches: + logger.warning("Could not find OpenSSL version; not disabling session tickets.") + return None + self._openssl_version = matches[0].decode('UTF-8') + return self._openssl_version + def prepare(self): """Prepare the authenticator/installer. @@ -271,8 +334,12 @@ class ApacheConfigurator(common.Installer): # Get all of the available vhosts self.vhosts = self.get_virtual_hosts() + # We may try to enable mod_ssl later. If so, we shouldn't warn if we can't find it now. + # This is currently only true for debian/ubuntu. + warn_on_no_mod_ssl = not self.option("handle_modules") self.install_ssl_options_conf(self.mod_ssl_conf, - self.updated_mod_ssl_conf_digest) + self.updated_mod_ssl_conf_digest, + warn_on_no_mod_ssl) # Prevent two Apache plugins from modifying a config at once try: @@ -1241,6 +1308,14 @@ class ApacheConfigurator(common.Installer): self.enable_mod("socache_shmcb", temp=temp) if "ssl_module" not in self.parser.modules: self.enable_mod("ssl", temp=temp) + # Make sure we're not throwing away any unwritten changes to the config + self.parser.ensure_augeas_state() + self.parser.aug.load() + self.parser.reset_modules() # Reset to load the new ssl_module path + # Call again because now we can gate on openssl version + self.install_ssl_options_conf(self.mod_ssl_conf, + self.updated_mod_ssl_conf_digest, + warn_on_no_mod_ssl=True) def make_vhost_ssl(self, nonssl_vhost): """Makes an ssl_vhost version of a nonssl_vhost. @@ -2454,14 +2529,19 @@ class ApacheConfigurator(common.Installer): self.restart() self.parser.reset_modules() - def install_ssl_options_conf(self, options_ssl, options_ssl_digest): - """Copy Certbot's SSL options file into the system's config dir if required.""" + def install_ssl_options_conf(self, options_ssl, options_ssl_digest, warn_on_no_mod_ssl=True): + """Copy Certbot's SSL options file into the system's config dir if required. + + :param bool warn_on_no_mod_ssl: True if we should warn if mod_ssl is not found. + """ # XXX if we ever try to enforce a local privilege boundary (eg, running # certbot for unprivileged users via setuid), this function will need # to be modified. - return common.install_version_controlled_file(options_ssl, options_ssl_digest, - self.option("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) + apache_config_path = self.pick_apache_config(warn_on_no_mod_ssl) + + return common.install_version_controlled_file( + options_ssl, options_ssl_digest, apache_config_path, constants.ALL_SSL_OPTIONS_HASHES) def enable_autohsts(self, _unused_lineage, domains): """ diff --git a/certbot-apache/certbot_apache/_internal/constants.py b/certbot-apache/certbot_apache/_internal/constants.py index a37bebac5..68d29406e 100644 --- a/certbot-apache/certbot_apache/_internal/constants.py +++ b/certbot-apache/certbot_apache/_internal/constants.py @@ -26,6 +26,7 @@ ALL_SSL_OPTIONS_HASHES = [ '06675349e457eae856120cdebb564efe546f0b87399f2264baeb41e442c724c7', '5cc003edd93fb9cd03d40c7686495f8f058f485f75b5e764b789245a386e6daf', '007cd497a56a3bb8b6a2c1aeb4997789e7e38992f74e44cc5d13a625a738ac73', + '34783b9e2210f5c4a23bced2dfd7ec289834716673354ed7c7abf69fe30192a3', ] """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC""" diff --git a/certbot-apache/certbot_apache/_internal/override_arch.py b/certbot-apache/certbot_apache/_internal/override_arch.py index 2765bd238..54202c087 100644 --- a/certbot-apache/certbot_apache/_internal/override_arch.py +++ b/certbot-apache/certbot_apache/_internal/override_arch.py @@ -1,9 +1,7 @@ """ Distribution specific override class for Arch Linux """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import configurator @@ -26,6 +24,4 @@ class ArchConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 2ab160c2f..0de882519 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -1,14 +1,12 @@ """ Distribution specific override class for CentOS family (RHEL, Fedora) """ import logging -import pkg_resources import zope.interface from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import os from certbot.errors import MisconfigurationError from certbot_apache._internal import apache_util from certbot_apache._internal import configurator @@ -37,8 +35,6 @@ class CentOSConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def config_test(self): diff --git a/certbot-apache/certbot_apache/_internal/override_darwin.py b/certbot-apache/certbot_apache/_internal/override_darwin.py index 00faff623..f19823866 100644 --- a/certbot-apache/certbot_apache/_internal/override_darwin.py +++ b/certbot-apache/certbot_apache/_internal/override_darwin.py @@ -1,9 +1,7 @@ """ Distribution specific override class for macOS """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import configurator @@ -26,6 +24,4 @@ class DarwinConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/other", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) diff --git a/certbot-apache/certbot_apache/_internal/override_debian.py b/certbot-apache/certbot_apache/_internal/override_debian.py index 77ced6a3f..c977fb43e 100644 --- a/certbot-apache/certbot_apache/_internal/override_debian.py +++ b/certbot-apache/certbot_apache/_internal/override_debian.py @@ -1,7 +1,6 @@ """ Distribution specific override class for Debian family (Ubuntu/Debian) """ import logging -import pkg_resources import zope.interface from certbot import errors @@ -34,8 +33,6 @@ class DebianConfigurator(configurator.ApacheConfigurator): handle_modules=True, handle_sites=True, challenge_location="/etc/apache2", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def enable_site(self, vhost): diff --git a/certbot-apache/certbot_apache/_internal/override_fedora.py b/certbot-apache/certbot_apache/_internal/override_fedora.py index 8197b0dcd..2436c76cc 100644 --- a/certbot-apache/certbot_apache/_internal/override_fedora.py +++ b/certbot-apache/certbot_apache/_internal/override_fedora.py @@ -1,11 +1,9 @@ """ Distribution specific override class for Fedora 29+ """ -import pkg_resources import zope.interface from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser @@ -31,9 +29,6 @@ class FedoraConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - # TODO: eventually newest version of Fedora will need their own config - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def config_test(self): diff --git a/certbot-apache/certbot_apache/_internal/override_gentoo.py b/certbot-apache/certbot_apache/_internal/override_gentoo.py index c215771e6..6b7416c0d 100644 --- a/certbot-apache/certbot_apache/_internal/override_gentoo.py +++ b/certbot-apache/certbot_apache/_internal/override_gentoo.py @@ -1,9 +1,7 @@ """ Distribution specific override class for Gentoo Linux """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser @@ -29,8 +27,6 @@ class GentooConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def _prepare_options(self): diff --git a/certbot-apache/certbot_apache/_internal/override_suse.py b/certbot-apache/certbot_apache/_internal/override_suse.py index 0c9219e6d..895e0cb05 100644 --- a/certbot-apache/certbot_apache/_internal/override_suse.py +++ b/certbot-apache/certbot_apache/_internal/override_suse.py @@ -1,9 +1,7 @@ """ Distribution specific override class for OpenSUSE """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import configurator @@ -26,6 +24,4 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 992672913..82d7df6aa 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -9,7 +9,6 @@ import six from acme.magic_typing import Dict from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util @@ -52,7 +51,7 @@ class ApacheParser(object): "version 1.2.0 or higher, please make sure you have you have " "those installed.") - self.modules = set() # type: Set[str] + self.modules = {} # type: Dict[str, str] self.parser_paths = {} # type: Dict[str, List[str]] self.variables = {} # type: Dict[str, str] @@ -249,14 +248,14 @@ class ApacheParser(object): def add_mod(self, mod_name): """Shortcut for updating parser modules.""" if mod_name + "_module" not in self.modules: - self.modules.add(mod_name + "_module") + self.modules[mod_name + "_module"] = None if "mod_" + mod_name + ".c" not in self.modules: - self.modules.add("mod_" + mod_name + ".c") + self.modules["mod_" + mod_name + ".c"] = None def reset_modules(self): """Reset the loaded modules list. This is called from cleanup to clear temporarily loaded modules.""" - self.modules = set() + self.modules = {} self.update_modules() self.parse_modules() @@ -267,7 +266,7 @@ class ApacheParser(object): the iteration issue. Else... parse and enable mods at same time. """ - mods = set() # type: Set[str] + mods = {} # type: Dict[str, str] matches = self.find_dir("LoadModule") iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration @@ -281,8 +280,8 @@ class ApacheParser(object): mod_name = self.get_arg(match_name) mod_filename = self.get_arg(match_filename) if mod_name and mod_filename: - mods.add(mod_name) - mods.add(os.path.basename(mod_filename)[:-2] + "c") + mods[mod_name] = mod_filename + mods[os.path.basename(mod_filename)[:-2] + "c"] = mod_filename else: logger.debug("Could not read LoadModule directive from Augeas path: %s", match_name[6:]) @@ -621,7 +620,7 @@ class ApacheParser(object): def exclude_dirs(self, matches): """Exclude directives that are not loaded into the configuration.""" - filters = [("ifmodule", self.modules), ("ifdefine", self.variables)] + filters = [("ifmodule", self.modules.keys()), ("ifdefine", self.variables)] valid_matches = [] @@ -662,6 +661,25 @@ class ApacheParser(object): return True + def standard_path_from_server_root(self, arg): + """Ensure paths are consistent and absolute + + :param str arg: Argument of directive + + :returns: Standardized argument path + :rtype: str + """ + # Remove beginning and ending quotes + arg = arg.strip("'\"") + + # Standardize the include argument based on server root + if not arg.startswith("/"): + # Normpath will condense ../ + arg = os.path.normpath(os.path.join(self.root, arg)) + else: + arg = os.path.normpath(arg) + return arg + def _get_include_path(self, arg): """Converts an Apache Include directive into Augeas path. @@ -682,16 +700,7 @@ class ApacheParser(object): # if matchObj.group() != arg: # logger.error("Error: Invalid regexp characters in %s", arg) # return [] - - # Remove beginning and ending quotes - arg = arg.strip("'\"") - - # Standardize the include argument based on server root - if not arg.startswith("/"): - # Normpath will condense ../ - arg = os.path.normpath(os.path.join(self.root, arg)) - else: - arg = os.path.normpath(arg) + arg = self.standard_path_from_server_root(arg) # Attempts to add a transform to the file if one does not already exist if os.path.isdir(arg): diff --git a/certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf b/certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf new file mode 100644 index 000000000..32a2c3335 --- /dev/null +++ b/certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf @@ -0,0 +1,19 @@ +# This file contains important security parameters. If you modify this file +# manually, Certbot will be unable to automatically provide future security +# updates. Instead, Certbot will print and log an error message with a path to +# the up-to-date file that you will need to refer to when manually updating +# this file. + +SSLEngine on + +# Intermediate configuration, tweak to your needs +SSLProtocol all -SSLv2 -SSLv3 -TLSv1 -TLSv1.1 +SSLCipherSuite ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 +SSLHonorCipherOrder off +SSLSessionTickets off + +SSLOptions +StrictRequire + +# Add vhost name to log entries: +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined +LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common diff --git a/certbot-apache/certbot_apache/_internal/options-ssl-apache.conf b/certbot-apache/certbot_apache/_internal/tls_configs/old-options-ssl-apache.conf similarity index 100% rename from certbot-apache/certbot_apache/_internal/options-ssl-apache.conf rename to certbot-apache/certbot_apache/_internal/tls_configs/old-options-ssl-apache.conf diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index c9901ecdb..8e4f15d6b 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -20,10 +20,10 @@ class AutoHSTSTest(util.ApacheTest): self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) - self.config.parser.modules.add("headers_module") - self.config.parser.modules.add("mod_headers.c") - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["headers_module"] = None + self.config.parser.modules["mod_headers.c"] = None + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") @@ -42,8 +42,8 @@ class AutoHSTSTest(util.ApacheTest): @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.restart") @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod") def test_autohsts_enable_headers_mod(self, mock_enable, _restart): - self.config.parser.modules.discard("headers_module") - self.config.parser.modules.discard("mod_header.c") + self.config.parser.modules.pop("headers_module", None) + self.config.parser.modules.pop("mod_header.c", None) self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) self.assertTrue(mock_enable.called) diff --git a/certbot-apache/tests/centos_test.py b/certbot-apache/tests/centos_test.py index 55fee3faa..5e334c51e 100644 --- a/certbot-apache/tests/centos_test.py +++ b/certbot-apache/tests/centos_test.py @@ -126,7 +126,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest): return mod_val return "" mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() + self.config.parser.modules = {} self.config.parser.variables = {} with mock.patch("certbot.util.get_os_info") as mock_osi: diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index cbb052155..54f4c9251 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -341,9 +341,9 @@ class MultipleVhostsTest(util.ApacheTest): def test_deploy_cert_enable_new_vhost(self): # Create ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.assertFalse(ssl_vhost.enabled) self.config.deploy_cert( @@ -377,9 +377,9 @@ class MultipleVhostsTest(util.ApacheTest): # pragma: no cover def test_deploy_cert(self): - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None # Patch _add_dummy_ssl_directives to make sure we write them correctly # pylint: disable=protected-access orig_add_dummy = self.config._add_dummy_ssl_directives @@ -459,9 +459,9 @@ class MultipleVhostsTest(util.ApacheTest): method is called with an invalid vhost parameter. Currently this tests that a PluginError is appropriately raised when important directives are missing in an SSL module.""" - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None def side_effect(*args): """Mocks case where an SSLCertificateFile directive can be found @@ -544,7 +544,8 @@ class MultipleVhostsTest(util.ApacheTest): call_found = True self.assertTrue(call_found) - def test_prepare_server_https(self): + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https(self, mock_reset): mock_enable = mock.Mock() self.config.enable_mod = mock_enable @@ -570,7 +571,8 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(mock_add_dir.call_count, 2) - def test_prepare_server_https_named_listen(self): + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https_named_listen(self, mock_reset): mock_find = mock.Mock() mock_find.return_value = ["test1", "test2", "test3"] mock_get = mock.Mock() @@ -608,7 +610,8 @@ class MultipleVhostsTest(util.ApacheTest): # self.config.prepare_server_https("8080", temp=True) # self.assertEqual(self.listens, 0) - def test_prepare_server_https_needed_listen(self): + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https_needed_listen(self, mock_reset): mock_find = mock.Mock() mock_find.return_value = ["test1", "test2"] mock_get = mock.Mock() @@ -624,8 +627,8 @@ class MultipleVhostsTest(util.ApacheTest): self.config.prepare_server_https("443") self.assertEqual(mock_add_dir.call_count, 1) - def test_prepare_server_https_mixed_listen(self): - + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https_mixed_listen(self, mock_reset): mock_find = mock.Mock() mock_find.return_value = ["test1", "test2"] mock_get = mock.Mock() @@ -904,7 +907,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache._internal.display_ops.select_vhost") @mock.patch("certbot.util.exe_exists") def test_enhance_unknown_vhost(self, mock_exe, mock_sel_vhost, mock_get): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None mock_exe.return_value = True ssl_vh1 = obj.VirtualHost( "fp1", "ap1", set([obj.Addr(("*", "443"))]), @@ -942,8 +945,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling(self, mock_exe): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True @@ -969,8 +972,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling_twice(self, mock_exe): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True @@ -997,8 +1000,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_ocsp_unsupported_apache_version(self, mock_exe): mock_exe.return_value = True self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.config.get_version = mock.Mock(return_value=(2, 2, 0)) self.config.choose_vhost("certbot.demo") @@ -1021,8 +1024,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_http_header_hsts(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None mock_exe.return_value = True # This will create an ssl vhost for certbot.demo @@ -1042,9 +1045,9 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(len(hsts_header), 4) def test_http_header_hsts_twice(self): - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None # skip the enable mod - self.config.parser.modules.add("headers_module") + self.config.parser.modules["headers_module"] = None # This will create an ssl vhost for encryption-example.demo self.config.choose_vhost("encryption-example.demo") @@ -1060,8 +1063,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_http_header_uir(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None mock_exe.return_value = True @@ -1084,9 +1087,9 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(len(uir_header), 4) def test_http_header_uir_twice(self): - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None # skip the enable mod - self.config.parser.modules.add("headers_module") + self.config.parser.modules["headers_module"] = None # This will create an ssl vhost for encryption-example.demo self.config.choose_vhost("encryption-example.demo") @@ -1101,7 +1104,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_well_formed_http(self, mock_exe, _): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2)) @@ -1127,7 +1130,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_rewrite_rule_exists(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.parser.add_dir( self.vh_truth[3].path, "RewriteRule", ["Unknown"]) @@ -1136,7 +1139,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_rewrite_engine_exists(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.parser.add_dir( self.vh_truth[3].path, "RewriteEngine", "on") @@ -1146,7 +1149,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_with_existing_rewrite(self, mock_exe, _): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2, 0)) @@ -1180,7 +1183,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_with_old_https_redirection(self, mock_exe, _): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2, 0)) @@ -1209,7 +1212,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_with_conflict(self): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None ssl_vh = obj.VirtualHost( "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]), @@ -1222,7 +1225,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_two_domains_one_vhost(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) # Creates ssl vhost for the domain @@ -1237,7 +1240,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_from_previous_run(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.choose_vhost("red.blue.purple.com") self.config.enhance("red.blue.purple.com", "redirect") @@ -1250,7 +1253,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enhance, "green.blue.purple.com", "redirect") def test_create_own_redirect(self): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) # For full testing... give names... self.vh_truth[1].name = "default.com" @@ -1261,7 +1264,7 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(len(self.config.vhosts), 13) def test_create_own_redirect_for_old_apache_version(self): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 2)) # For full testing... give names... self.vh_truth[1].name = "default.com" @@ -1326,9 +1329,9 @@ class MultipleVhostsTest(util.ApacheTest): def test_deploy_cert_not_parsed_path(self): # Make sure that we add include to root config for vhosts when # handle-sites is false - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None tmp_path = filesystem.realpath(tempfile.mkdtemp("vhostroot")) filesystem.chmod(tmp_path, 0o755) mock_p = "certbot_apache._internal.configurator.ApacheConfigurator._get_ssl_vhost_path" @@ -1441,8 +1444,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator._choose_vhosts_wildcard") def test_enhance_wildcard_after_install(self, mock_choose): # pylint: disable=protected-access - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None self.vh_truth[3].ssl = True self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] self.config.enhance("*.certbot.demo", "ensure-http-header", @@ -1453,8 +1456,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_enhance_wildcard_no_install(self, mock_choose): self.vh_truth[3].ssl = True mock_choose.return_value = [self.vh_truth[3]] - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None self.config.enhance("*.certbot.demo", "ensure-http-header", "Upgrade-Insecure-Requests") self.assertTrue(mock_choose.called) @@ -1638,7 +1641,7 @@ class MultiVhostsTest(util.ApacheTest): @certbot_util.patch_get_utility() def test_make_vhost_ssl_with_existing_rewrite_rule(self, mock_get_utility): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[4]) @@ -1658,7 +1661,7 @@ class MultiVhostsTest(util.ApacheTest): @certbot_util.patch_get_utility() def test_make_vhost_ssl_with_existing_rewrite_conds(self, mock_get_utility): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[3]) @@ -1700,7 +1703,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.config.updated_mod_ssl_conf_digest) def _current_ssl_options_hash(self): - return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC")) + return crypto_util.sha256sum(self.config.pick_apache_config()) def _assert_current_file(self): self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) @@ -1736,7 +1739,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.assertFalse(mock_logger.warning.called) self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) self.assertEqual(crypto_util.sha256sum( - self.config.option("MOD_SSL_CONF_SRC")), + self.config.pick_apache_config()), self._current_ssl_options_hash()) self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf), self._current_ssl_options_hash()) @@ -1752,19 +1755,99 @@ class InstallSslOptionsConfTest(util.ApacheTest): "%s has been manually modified; updated file " "saved to %s. We recommend updating %s for security purposes.") self.assertEqual(crypto_util.sha256sum( - self.config.option("MOD_SSL_CONF_SRC")), + self.config.pick_apache_config()), self._current_ssl_options_hash()) # only print warning once with mock.patch("certbot.plugins.common.logger") as mock_logger: self._call() self.assertFalse(mock_logger.warning.called) - def test_current_file_hash_in_all_hashes(self): + def test_ssl_config_files_hash_in_all_hashes(self): + """ + It is really critical that all TLS Apache config files have their SHA256 hash registered in + constants.ALL_SSL_OPTIONS_HASHES. Otherwise Certbot will mistakenly assume that the config + file has been manually edited by the user, and will refuse to update it. + This test ensures that all necessary hashes are present. + """ from certbot_apache._internal.constants import ALL_SSL_OPTIONS_HASHES - self.assertTrue(self._current_ssl_options_hash() in ALL_SSL_OPTIONS_HASHES, - "Constants.ALL_SSL_OPTIONS_HASHES must be appended" - " with the sha256 hash of self.config.mod_ssl_conf when it is updated.") + import pkg_resources + tls_configs_dir = pkg_resources.resource_filename( + "certbot_apache", os.path.join("_internal", "tls_configs")) + all_files = [os.path.join(tls_configs_dir, name) for name in os.listdir(tls_configs_dir) + if name.endswith('options-ssl-apache.conf')] + self.assertTrue(all_files) + for one_file in all_files: + file_hash = crypto_util.sha256sum(one_file) + self.assertTrue(file_hash in ALL_SSL_OPTIONS_HASHES, + "Constants.ALL_SSL_OPTIONS_HASHES must be appended with the sha256 " + "hash of {0} when it is updated.".format(one_file)) + + def test_openssl_version(self): + self.config._openssl_version = None + some_string_contents = b""" + SSLOpenSSLConfCmd + OpenSSL configuration command + SSLv3 not supported by this version of OpenSSL + '%s': invalid OpenSSL configuration command + OpenSSL 1.0.2g 1 Mar 2016 + OpenSSL + AH02407: "SSLOpenSSLConfCmd %s %s" failed for %s + AH02556: "SSLOpenSSLConfCmd %s %s" applied to %s + OpenSSL 1.0.2g 1 Mar 2016 + """ + self.config.parser.modules['ssl_module'] = '/fake/path' + with mock.patch("certbot_apache._internal.configurator." + "ApacheConfigurator._open_module_file") as mock_omf: + mock_omf.return_value = some_string_contents + self.assertEqual(self.config.openssl_version(), "1.0.2g") + + def test_current_version(self): + self.config.version = (2, 4, 10) + self.config._openssl_version = '1.0.2m' + self.assertTrue('old' in self.config.pick_apache_config()) + + self.config.version = (2, 4, 11) + self.config._openssl_version = '1.0.2m' + self.assertTrue('current' in self.config.pick_apache_config()) + + self.config._openssl_version = '1.0.2a' + self.assertTrue('old' in self.config.pick_apache_config()) + + def test_openssl_version_warns(self): + self.config._openssl_version = '1.0.2a' + self.assertEqual(self.config.openssl_version(), '1.0.2a') + + self.config._openssl_version = None + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Could not find ssl_module" in mock_log.call_args[0][0]) + + self.config._openssl_version = None + self.config.parser.modules['ssl_module'] = None + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Could not find ssl_module" in mock_log.call_args[0][0]) + + self.config.parser.modules['ssl_module'] = "/fake/path" + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + # Check that correct logger.warning was printed + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Unable to read" in mock_log.call_args[0][0]) + + contents_missing_openssl = b"these contents won't match the regex" + with mock.patch("certbot_apache._internal.configurator." + "ApacheConfigurator._open_module_file") as mock_omf: + mock_omf.return_value = contents_missing_openssl + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + # Check that correct logger.warning was printed + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Could not find OpenSSL" in mock_log.call_args[0][0]) + + def test_open_module_file(self): + mock_open = mock.mock_open(read_data="testing 12 3") + with mock.patch("six.moves.builtins.open", mock_open): + self.assertEqual(self.config._open_module_file("/nonsense/"), "testing 12 3") if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/tests/debian_test.py b/certbot-apache/tests/debian_test.py index 400e503fb..7a65bb7b1 100644 --- a/certbot-apache/tests/debian_test.py +++ b/certbot-apache/tests/debian_test.py @@ -61,8 +61,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): def test_deploy_cert_enable_new_vhost(self): # Create ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None self.assertFalse(ssl_vhost.enabled) self.config.deploy_cert( "encryption-example.demo", "example/cert.pem", "example/key.pem", @@ -92,8 +92,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) self.config = self.mock_deploy_cert(self.config) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] @@ -128,8 +128,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) self.config = self.mock_deploy_cert(self.config) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] @@ -143,8 +143,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 7)) self.config = self.mock_deploy_cert(self.config) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] @@ -157,7 +157,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling_enable_mod(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True # This will create an ssl vhost for certbot.demo @@ -169,7 +169,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ensure_http_header_enable_mod(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None mock_exe.return_value = True # This will create an ssl vhost for certbot.demo diff --git a/certbot-apache/tests/fedora_test.py b/certbot-apache/tests/fedora_test.py index cb1614278..7f1d6526f 100644 --- a/certbot-apache/tests/fedora_test.py +++ b/certbot-apache/tests/fedora_test.py @@ -120,7 +120,7 @@ class MultipleVhostsTestFedora(util.ApacheTest): return mod_val return "" mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() + self.config.parser.modules = {} self.config.parser.variables = {} with mock.patch("certbot.util.get_os_info") as mock_osi: diff --git a/certbot-apache/tests/gentoo_test.py b/certbot-apache/tests/gentoo_test.py index fb5d192d0..3b35dde54 100644 --- a/certbot-apache/tests/gentoo_test.py +++ b/certbot-apache/tests/gentoo_test.py @@ -117,7 +117,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest): return mod_val return None # pragma: no cover mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() + self.config.parser.modules = {} with mock.patch("certbot.util.get_os_info") as mock_osi: # Make sure we have the have the Gentoo httpd constants diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 85b17ca28..7d9019702 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -40,8 +40,8 @@ class ApacheHttp01Test(util.ApacheTest): modules = ["ssl", "rewrite", "authz_core", "authz_host"] for mod in modules: - self.config.parser.modules.add("mod_{0}.c".format(mod)) - self.config.parser.modules.add(mod + "_module") + self.config.parser.modules["mod_{0}.c".format(mod)] = None + self.config.parser.modules[mod + "_module"] = None from certbot_apache._internal.http_01 import ApacheHttp01 self.http = ApacheHttp01(self.config) @@ -52,24 +52,24 @@ class ApacheHttp01Test(util.ApacheTest): @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod") def test_enable_modules_apache_2_2(self, mock_enmod): self.config.version = (2, 2) - self.config.parser.modules.remove("authz_host_module") - self.config.parser.modules.remove("mod_authz_host.c") + del self.config.parser.modules["authz_host_module"] + del self.config.parser.modules["mod_authz_host.c"] enmod_calls = self.common_enable_modules_test(mock_enmod) self.assertEqual(enmod_calls[0][0][0], "authz_host") @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod") def test_enable_modules_apache_2_4(self, mock_enmod): - self.config.parser.modules.remove("authz_core_module") - self.config.parser.modules.remove("mod_authz_core.c") + del self.config.parser.modules["authz_core_module"] + del self.config.parser.modules["mod_authz_host.c"] enmod_calls = self.common_enable_modules_test(mock_enmod) self.assertEqual(enmod_calls[0][0][0], "authz_core") def common_enable_modules_test(self, mock_enmod): """Tests enabling mod_rewrite and other modules.""" - self.config.parser.modules.remove("rewrite_module") - self.config.parser.modules.remove("mod_rewrite.c") + del self.config.parser.modules["rewrite_module"] + del self.config.parser.modules["mod_rewrite.c"] self.http.prepare_http01_modules() diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py index f5a0a3d11..299eb4567 100644 --- a/certbot-apache/tests/parser_test.py +++ b/certbot-apache/tests/parser_test.py @@ -114,7 +114,7 @@ class BasicParserTest(util.ParserTest): """ from certbot_apache._internal.parser import get_aug_path # This makes sure that find_dir will work - self.parser.modules.add("mod_ssl.c") + self.parser.modules["mod_ssl.c"] = "/fake/path" self.parser.add_dir_to_ifmodssl( get_aug_path(self.parser.loc["default"]), @@ -128,7 +128,7 @@ class BasicParserTest(util.ParserTest): def test_add_dir_to_ifmodssl_multiple(self): from certbot_apache._internal.parser import get_aug_path # This makes sure that find_dir will work - self.parser.modules.add("mod_ssl.c") + self.parser.modules["mod_ssl.c"] = "/fake/path" self.parser.add_dir_to_ifmodssl( get_aug_path(self.parser.loc["default"]), @@ -260,7 +260,7 @@ class BasicParserTest(util.ParserTest): expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443", "example_path": "Documents/path"} - self.parser.modules = set() + self.parser.modules = {} with mock.patch( "certbot_apache._internal.parser.ApacheParser.parse_file") as mock_parse: self.parser.update_runtime_variables() @@ -282,7 +282,7 @@ class BasicParserTest(util.ParserTest): os.path.dirname(self.parser.loc["root"])) mock_cfg.return_value = inc_val - self.parser.modules = set() + self.parser.modules = {} with mock.patch( "certbot_apache._internal.parser.ApacheParser.parse_file") as mock_parse: diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index ccd0b274d..4994103f3 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -85,7 +85,8 @@ def get_apache_configurator( config_dir, work_dir, version=(2, 4, 7), os_info="generic", conf_vhost_path=None, - use_parsernode=False): + use_parsernode=False, + openssl_version="1.1.1a"): """Create an Apache Configurator with the specified options. :param conf: Function that returns binary paths. self.conf in Configurator @@ -118,7 +119,8 @@ def get_apache_configurator( except KeyError: config_class = configurator.ApacheConfigurator config = config_class(config=mock_le_config, name="apache", - version=version, use_parsernode=use_parsernode) + version=version, use_parsernode=use_parsernode, + openssl_version=openssl_version) if not conf_vhost_path: config_class.OS_DEFAULTS["vhost_root"] = vhost_path else: diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index c35af0d0f..2f551c8ab 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -6,6 +6,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added +* Turn off session tickets for apache plugin by default when appropriate. * Added serial number of certificate to the output of `certbot certificates` * Expose two new environment variables in the authenticator and cleanup scripts used by the `manual` plugin: `CERTBOT_REMAINING_CHALLENGES` is equal to the number of challenges @@ -31,9 +32,9 @@ More details about these changes can be found on our GitHub repo. ### Added -* Added certbot.ocsp Certbot's API. The certbot.ocsp module can be used to +* Added certbot.ocsp Certbot's API. The certbot.ocsp module can be used to determine the OCSP status of certificates. -* Don't verify the existing certificate in HTTP01Response.simple_verify, for +* Don't verify the existing certificate in HTTP01Response.simple_verify, for compatibility with the real-world ACME challenge checks. * Added support for `$hostname` in nginx `server_name` directive @@ -42,7 +43,7 @@ More details about these changes can be found on our GitHub repo. * Certbot will now renew certificates early if they have been revoked according to OCSP. * Fix acme module warnings when response Content-Type includes params (e.g. charset). -* Fixed issue where webroot plugin would incorrectly raise `Read-only file system` +* Fixed issue where webroot plugin would incorrectly raise `Read-only file system` error when creating challenge directories (issue #7165). ### Fixed diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 9af39e8bb..9c3b95a31 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -58,6 +58,22 @@ if [ $? -ne 0 ] ; then FAIL=1 fi +# Check that ssl_module detection is working on various systems +if [ "$OS_TYPE" = "ubuntu" ] ; then + MOD_SSL_LOCATION="/usr/lib/apache2/modules/mod_ssl.so" + APACHE_NAME=apache2ctl +elif [ "$OS_TYPE" = "centos" ]; then + MOD_SSL_LOCATION="/etc/httpd/modules/mod_ssl.so" + APACHE_NAME=httpd +fi +OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9) +APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8) +"$PYTHON_NAME" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" +if [ $? -ne 0 ] ; then + FAIL=1 +fi + + if [ "$OS_TYPE" = "ubuntu" ] ; then export SERVER="$BOULDER_URL" "$VENV_PATH/bin/tox" -e apacheconftest diff --git a/tests/letstest/scripts/test_openssl_version.py b/tests/letstest/scripts/test_openssl_version.py new file mode 100644 index 000000000..c55441c5d --- /dev/null +++ b/tests/letstest/scripts/test_openssl_version.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# Test script for OpenSSL version checking +from distutils.version import LooseVersion +import sys + + +def main(openssl_version, apache_version): + if not openssl_version.strip(): + raise Exception("No OpenSSL version found.") + if not apache_version.strip(): + raise Exception("No Apache version found.") + conf_file_location = "/etc/letsencrypt/options-ssl-apache.conf" + with open(conf_file_location) as f: + contents = f.read() + if LooseVersion(apache_version.strip()) < LooseVersion('2.4.11') or \ + LooseVersion(openssl_version.strip()) < LooseVersion('1.0.2l'): + # should be old version + # assert SSLSessionTickets not in conf file + if "SSLSessionTickets" in contents: + raise Exception("Apache or OpenSSL version is too old, " + "but SSLSessionTickets is set.") + else: + # should be current version + # assert SSLSessionTickets in conf file + if "SSLSessionTickets" not in contents: + raise Exception("Apache and OpenSSL versions are sufficiently new, " + "but SSLSessionTickets is not set.") + +if __name__ == '__main__': + main(*sys.argv[1:]) From 1285297b23e445c10dfac481b02447d4cc2cf6f6 Mon Sep 17 00:00:00 2001 From: m0namon <51840727+m0namon@users.noreply.github.com> Date: Mon, 23 Mar 2020 17:05:22 -0700 Subject: [PATCH 62/92] [Apache v2] Load apacheconfig tree and gate related tests (#7710) * Load apacheconfig dependency, gate behind flag * Bump apacheconfig dependency to latest version and install dev version of apache for coverage tests * Move augeasnode_test tests to more generic parsernode_test * Revert "Move augeasnode_test tests to more generic parsernode_test" This reverts commit 6bb986ef786b9d68bb72776bde66e6572cf505a9. * Mock AugeasNode into DualNode's place, and run augeasnode tests exclusively on AugeasNode * Don't calculate coverage for skeleton functions * clean up helper function in augeasnode_test --- .../certbot_apache/_internal/apacheparser.py | 11 ++-- .../certbot_apache/_internal/configurator.py | 23 +++++-- certbot-apache/setup.py | 2 +- certbot-apache/tests/augeasnode_test.py | 65 ++++++++++++------- .../tests/parsernode_configurator_test.py | 7 ++ tools/dev_constraints.txt | 2 +- tools/oldest_constraints.txt | 2 +- tox.ini | 2 + 8 files changed, 76 insertions(+), 38 deletions(-) diff --git a/certbot-apache/certbot_apache/_internal/apacheparser.py b/certbot-apache/certbot_apache/_internal/apacheparser.py index 77f4517fe..c7b723ae6 100644 --- a/certbot-apache/certbot_apache/_internal/apacheparser.py +++ b/certbot-apache/certbot_apache/_internal/apacheparser.py @@ -73,7 +73,7 @@ class ApacheDirectiveNode(ApacheParserNode): self.metadata == other.metadata) return False - def set_parameters(self, _parameters): + def set_parameters(self, _parameters): # pragma: no cover """Sets the parameters for DirectiveNode""" return @@ -97,7 +97,8 @@ class ApacheBlockNode(ApacheDirectiveNode): self.metadata == other.metadata) return False - def add_child_block(self, name, parameters=None, position=None): # pylint: disable=unused-argument + # pylint: disable=unused-argument + def add_child_block(self, name, parameters=None, position=None): # pragma: no cover """Adds a new BlockNode to the sequence of children""" new_block = ApacheBlockNode(name=assertions.PASS, parameters=assertions.PASS, @@ -107,7 +108,8 @@ class ApacheBlockNode(ApacheDirectiveNode): self.children += (new_block,) return new_block - def add_child_directive(self, name, parameters=None, position=None): # pylint: disable=unused-argument + # pylint: disable=unused-argument + def add_child_directive(self, name, parameters=None, position=None): # pragma: no cover """Adds a new DirectiveNode to the sequence of children""" new_dir = ApacheDirectiveNode(name=assertions.PASS, parameters=assertions.PASS, @@ -144,7 +146,8 @@ class ApacheBlockNode(ApacheDirectiveNode): filepath=assertions.PASS, metadata=self.metadata)] - def find_comments(self, comment, exact=False): # pylint: disable=unused-argument + # pylint: disable=unused-argument + def find_comments(self, comment, exact=False): # pragma: no cover """Recursive search of DirectiveNodes from the sequence of children""" return [ApacheCommentNode(comment=assertions.PASS, ancestor=self, diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index fc386404d..86d579954 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -12,6 +12,11 @@ import time import six import zope.component import zope.interface +try: + import apacheconfig + HAS_APACHECONFIG = True +except ImportError: # pragma: no cover + HAS_APACHECONFIG = False from acme import challenges from acme.magic_typing import DefaultDict @@ -430,11 +435,17 @@ class ApacheConfigurator(common.Installer): def get_parsernode_root(self, metadata): """Initializes the ParserNode parser root instance.""" - apache_vars = dict() - apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) - apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) - apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) - metadata["apache_vars"] = apache_vars + if HAS_APACHECONFIG: + apache_vars = dict() + apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) + apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) + apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) + metadata["apache_vars"] = apache_vars + + with open(self.parser.loc["root"]) as f: + with apacheconfig.make_loader(writable=True, + **apacheconfig.flavors.NATIVE_APACHE) as loader: + metadata["ac_ast"] = loader.loads(f.read()) return dualparser.DualBlockNode( name=assertions.PASS, @@ -974,7 +985,7 @@ class ApacheConfigurator(common.Installer): """ v1_vhosts = self.get_virtual_hosts_v1() - if self.USE_PARSERNODE: + if self.USE_PARSERNODE and HAS_APACHECONFIG: v2_vhosts = self.get_virtual_hosts_v2() for v1_vh in v1_vhosts: diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 4ec1d0a9c..14300370a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -19,7 +19,7 @@ install_requires = [ ] dev_extras = [ - 'apacheconfig>=0.3.1', + 'apacheconfig>=0.3.2', ] class PyTest(TestCommand): diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 8417bc283..4468a838e 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -1,13 +1,25 @@ """Tests for AugeasParserNode classes""" import mock +import os +import unittest import util from certbot import errors from certbot_apache._internal import assertions +from certbot_apache._internal import augeasparser +def _get_augeasnode_mock(filepath): + """ Helper function for mocking out DualNode instance with an AugeasNode """ + def augeasnode_mock(metadata): + return augeasparser.AugeasBlockNode( + name=assertions.PASS, + ancestor=None, + filepath=filepath, + metadata=metadata) + return augeasnode_mock class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public-methods """Test AugeasParserNode using available test configurations""" @@ -15,8 +27,11 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- def setUp(self): # pylint: disable=arguments-differ super(AugeasParserNodeTest, self).setUp() - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, use_parsernode=True) + with mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.get_parsernode_root") as mock_parsernode: + mock_parsernode.side_effect = _get_augeasnode_mock( + os.path.join(self.config_path, "apache2.conf")) + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, use_parsernode=True) self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") @@ -110,7 +125,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- name=servernames[0].name, parameters=["test", "setting", "these"], ancestor=assertions.PASS, - metadata=servernames[0].primary.metadata + metadata=servernames[0].metadata ) self.assertTrue(mock_set.called) self.assertEqual( @@ -145,26 +160,26 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.assertTrue(found) def test_add_child_comment(self): - newc = self.config.parser_root.primary.add_child_comment("The content") + newc = self.config.parser_root.add_child_comment("The content") comments = self.config.parser_root.find_comments("The content") self.assertEqual(len(comments), 1) self.assertEqual( newc.metadata["augeaspath"], - comments[0].primary.metadata["augeaspath"] + comments[0].metadata["augeaspath"] ) self.assertEqual(newc.comment, comments[0].comment) def test_delete_child(self): - listens = self.config.parser_root.primary.find_directives("Listen") + listens = self.config.parser_root.find_directives("Listen") self.assertEqual(len(listens), 1) - self.config.parser_root.primary.delete_child(listens[0]) + self.config.parser_root.delete_child(listens[0]) - listens = self.config.parser_root.primary.find_directives("Listen") + listens = self.config.parser_root.find_directives("Listen") self.assertEqual(len(listens), 0) def test_delete_child_not_found(self): listen = self.config.parser_root.find_directives("Listen")[0] - listen.primary.metadata["augeaspath"] = "/files/something/nonexistent" + listen.metadata["augeaspath"] = "/files/something/nonexistent" self.assertRaises( errors.PluginError, @@ -177,10 +192,10 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "NewBlock", ["first", "second"] ) - rpath, _, directive = nb.primary.metadata["augeaspath"].rpartition("/") + rpath, _, directive = nb.metadata["augeaspath"].rpartition("/") self.assertEqual( rpath, - self.config.parser_root.primary.metadata["augeaspath"] + self.config.parser_root.metadata["augeaspath"] ) self.assertTrue(directive.startswith("NewBlock")) @@ -189,8 +204,8 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "Beginning", position=0 ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Get first child first = parser.aug.match("{}/*[1]".format(root_path)) self.assertTrue(first[0].endswith("Beginning")) @@ -199,8 +214,8 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.config.parser_root.add_child_block( "VeryLast", ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Get last child last = parser.aug.match("{}/*[last()]".format(root_path)) self.assertTrue(last[0].endswith("VeryLast")) @@ -210,8 +225,8 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "VeryLastAlt", position=99999 ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Get last child last = parser.aug.match("{}/*[last()]".format(root_path)) self.assertTrue(last[0].endswith("VeryLastAlt")) @@ -221,15 +236,15 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "Middle", position=5 ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Augeas indices start at 1 :( middle = parser.aug.match("{}/*[6]".format(root_path)) self.assertTrue(middle[0].endswith("Middle")) def test_add_child_block_existing_name(self): - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # There already exists a single VirtualHost in the base config new_block = parser.aug.match("{}/VirtualHost[2]".format(root_path)) self.assertEqual(len(new_block), 0) @@ -238,7 +253,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- ) new_block = parser.aug.match("{}/VirtualHost[2]".format(root_path)) self.assertEqual(len(new_block), 1) - self.assertTrue(vh.primary.metadata["augeaspath"].endswith("VirtualHost[2]")) + self.assertTrue(vh.metadata["augeaspath"].endswith("VirtualHost[2]")) def test_node_init_error_bad_augeaspath(self): from certbot_apache._internal.augeasparser import AugeasBlockNode @@ -283,7 +298,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.assertEqual(len(dirs), 1) self.assertEqual(dirs[0].parameters, ("with", "parameters")) # The new directive was added to the very first line of the config - self.assertTrue(dirs[0].primary.metadata["augeaspath"].endswith("[1]")) + self.assertTrue(dirs[0].metadata["augeaspath"].endswith("[1]")) def test_add_child_directive_exception(self): self.assertRaises( @@ -313,6 +328,6 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.assertTrue(nonmacro_test) def test_find_ancestors_bad_path(self): - self.config.parser_root.primary.metadata["augeaspath"] = "" - ancs = self.config.parser_root.primary.find_ancestors("Anything") + self.config.parser_root.metadata["augeaspath"] = "" + ancs = self.config.parser_root.find_ancestors("Anything") self.assertEqual(len(ancs), 0) diff --git a/certbot-apache/tests/parsernode_configurator_test.py b/certbot-apache/tests/parsernode_configurator_test.py index 67d65995a..afaaa6e43 100644 --- a/certbot-apache/tests/parsernode_configurator_test.py +++ b/certbot-apache/tests/parsernode_configurator_test.py @@ -5,7 +5,14 @@ import mock import util +try: + import apacheconfig + HAS_APACHECONFIG = True +except ImportError: # pragma: no cover + HAS_APACHECONFIG = False + +@unittest.skipIf(not HAS_APACHECONFIG, reason='Tests require apacheconfig dependency') class ConfiguratorParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public-methods """Test AugeasParserNode using available test configurations""" diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index cfa036435..6e692841b 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -3,7 +3,7 @@ # Some dev package versions specified here may be overridden by higher level constraints # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). alabaster==0.7.10 -apacheconfig==0.3.1 +apacheconfig==0.3.2 apipkg==1.4 appnope==0.1.0 asn1crypto==0.22.0 diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 85d058796..402f3fef1 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -39,7 +39,7 @@ pytz==2012rc0 google-api-python-client==1.5.5 # Our setup.py constraints -apacheconfig==0.3.1 +apacheconfig==0.3.2 cloudflare==1.5.1 cryptography==1.2.3 parsedatetime==1.3 diff --git a/tox.ini b/tox.ini index 3903cdf45..8aa4bfbf2 100644 --- a/tox.ini +++ b/tox.ini @@ -120,12 +120,14 @@ setenv = basepython = python2.7 commands = {[base]install_packages} + {[base]pip_install} certbot-apache[dev] python tox.cover.py [testenv:py37-cover] basepython = python3.7 commands = {[base]install_packages} + {[base]pip_install} certbot-apache[dev] python tox.cover.py [testenv:lint] From e4a0edc7af05443448082a771e5cb981fa6544ed Mon Sep 17 00:00:00 2001 From: alexzorin Date: Wed, 25 Mar 2020 01:02:53 +1100 Subject: [PATCH 63/92] Add a 10-second timeout to OCSP queries. (#7860) * Add a 10-second timeout to OCSP queries. Closes #7859 * Update CHANGELOG * Fix test --- certbot/CHANGELOG.md | 1 + certbot/certbot/ocsp.py | 23 +++++++++++++---------- certbot/tests/ocsp_test.py | 4 ++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 2f551c8ab..c6f6c95a3 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -25,6 +25,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). `resource` field in any requests or the `type` field when responding to challenges. * Fix nginx plugin crash when non-ASCII configuration file is being read (instead, the user will be warned that UTF-8 must be used). +* Fix hanging OCSP queries during revocation checking - added a 10 second timeout. More details about these changes can be found on our GitHub repo. diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 9799c675c..1d5611b64 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -70,12 +70,13 @@ class RevocationChecker(object): """ return self.ocsp_revoked_by_paths(cert.cert_path, cert.chain_path) - def ocsp_revoked_by_paths(self, cert_path, chain_path): - # type: (str, str) -> bool + def ocsp_revoked_by_paths(self, cert_path, chain_path, timeout=10): + # type: (str, str, int) -> bool """Performs the OCSP revocation check :param str cert_path: Certificate filepath - :param str chain_path: Certificate chain filepath + :param str chain_path: Certificate chain + :param int timeout: Timeout (in seconds) for the OCSP query :returns: True if revoked; False if valid or the check failed or cert is expired. :rtype: bool @@ -96,11 +97,11 @@ class RevocationChecker(object): return False if self.use_openssl_binary: - return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url) - return _check_ocsp_cryptography(cert_path, chain_path, url) + return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url, timeout) + return _check_ocsp_cryptography(cert_path, chain_path, url, timeout) - def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url): - # type: (str, str, str, str) -> bool + def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url, timeout): + # type: (str, str, str, str, int) -> bool # jdkasten thanks "Bulletproof SSL and TLS - Ivan Ristic" for documenting this! cmd = ["openssl", "ocsp", "-no_nonce", @@ -110,6 +111,7 @@ class RevocationChecker(object): "-CAfile", chain_path, "-verify_other", chain_path, "-trust_other", + "-timeout", str(timeout), "-header"] + self.host_args(host) logger.debug("Querying OCSP for %s", cert_path) logger.debug(" ".join(cmd)) @@ -152,8 +154,8 @@ def _determine_ocsp_server(cert_path): return None, None -def _check_ocsp_cryptography(cert_path, chain_path, url): - # type: (str, str, str) -> bool +def _check_ocsp_cryptography(cert_path, chain_path, url, timeout): + # type: (str, str, str, int) -> bool # Retrieve OCSP response with open(chain_path, 'rb') as file_handler: issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) @@ -165,7 +167,8 @@ def _check_ocsp_cryptography(cert_path, chain_path, url): request_binary = request.public_bytes(serialization.Encoding.DER) try: response = requests.post(url, data=request_binary, - headers={'Content-Type': 'application/ocsp-request'}) + headers={'Content-Type': 'application/ocsp-request'}, + timeout=timeout) except requests.exceptions.RequestException: logger.info("OCSP check failed for %s (are we offline?)", cert_path, exc_info=True) return False diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 6b05a8d3c..9eb49e115 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -162,11 +162,11 @@ class OSCPTestCryptography(unittest.TestCase): @mock.patch('certbot.ocsp._determine_ocsp_server') @mock.patch('certbot.ocsp._check_ocsp_cryptography') - def test_ensure_cryptography_toggled(self, mock_revoke, mock_determine): + def test_ensure_cryptography_toggled(self, mock_check, mock_determine): mock_determine.return_value = ('http://example.com', 'example.com') self.checker.ocsp_revoked(self.cert_obj) - mock_revoke.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com') + mock_check.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com', 10) def test_revoke(self): with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL): From 6df90d17aeb3865ceda8ae30372cb1416a1a6c3b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 25 Mar 2020 14:50:13 -0700 Subject: [PATCH 64/92] Wait 5 minutes for boulder to start. (#7864) --- certbot-ci/certbot_integration_tests/utils/acme_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index b14bd4a32..4b20d5acc 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -170,7 +170,7 @@ class ACMEServer(object): # Wait for the ACME CA server to be up. print('=> Waiting for boulder instance to respond...') - misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=240) + misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300) # Configure challtestsrv to answer any A record request with ip of the docker host. response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), From dbda499b08722da8311fb2352bf423318772192e Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 31 Mar 2020 12:12:14 -0700 Subject: [PATCH 65/92] Remove interactive redirect ask (#7861) Fixes #7594. Removes the code asking interactively if the user would like to add a redirect. * Remove interactive redirect ask * display.enhancements is no longer used, so remove it. * update changelog * remove references to removed display.enhancements * add redirect_default flag to enhance_config to conditionally set default for redirect value * Update default in help text. --- certbot/CHANGELOG.md | 2 +- certbot/certbot/_internal/cli/__init__.py | 6 +- certbot/certbot/_internal/client.py | 14 ++--- .../certbot/_internal/display/enhancements.py | 63 ------------------- certbot/certbot/_internal/main.py | 2 +- certbot/tests/client_test.py | 19 ++---- certbot/tests/display/enhancements_test.py | 57 ----------------- 7 files changed, 14 insertions(+), 149 deletions(-) delete mode 100644 certbot/certbot/_internal/display/enhancements.py delete mode 100644 certbot/tests/display/enhancements_test.py diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index c6f6c95a3..88a4ab190 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -17,7 +17,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* +* Stop asking interactively if the user would like to add a redirect. ### Fixed diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index 96dfb163e..2f83edcc8 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -321,12 +321,14 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): "--redirect", action="store_true", dest="redirect", default=flag_default("redirect"), help="Automatically redirect all HTTP traffic to HTTPS for the newly " - "authenticated vhost. (default: Ask)") + "authenticated vhost. (default: redirect enabled for install and run, " + "disabled for enhance)") helpful.add( "security", "--no-redirect", action="store_false", dest="redirect", default=flag_default("redirect"), help="Do not automatically redirect all HTTP traffic to HTTPS for the newly " - "authenticated vhost. (default: Ask)") + "authenticated vhost. (default: redirect enabled for install and run, " + "disabled for enhance)") helpful.add( ["security", "enhance"], "--hsts", action="store_true", dest="hsts", default=flag_default("hsts"), diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index 526f4200f..a9bf946cc 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -28,7 +28,6 @@ from certbot._internal import constants from certbot._internal import eff from certbot._internal import error_handler from certbot._internal import storage -from certbot._internal.display import enhancements from certbot._internal.plugins import selection as plugin_selection from certbot.compat import os from certbot.display import ops as display_ops @@ -521,12 +520,13 @@ class Client(object): # sites may have been enabled / final cleanup self.installer.restart() - def enhance_config(self, domains, chain_path, ask_redirect=True): + def enhance_config(self, domains, chain_path, redirect_default=True): """Enhance the configuration. :param list domains: list of domains to configure :param chain_path: chain file path :type chain_path: `str` or `None` + :param redirect_default: boolean value that the "redirect" flag should default to :raises .errors.Error: if no installer is specified in the client. @@ -548,14 +548,8 @@ class Client(object): for config_name, enhancement_name, option in enhancement_info: config_value = getattr(self.config, config_name) if enhancement_name in supported: - if ask_redirect: - if config_name == "redirect" and config_value is None: - config_value = enhancements.ask(enhancement_name) - if not config_value: - logger.warning("Future versions of Certbot will automatically " - "configure the webserver so that all requests redirect to secure " - "HTTPS access. You can control this behavior and disable this " - "warning with the --redirect and --no-redirect flags.") + if config_name == "redirect" and config_value is None: + config_value = redirect_default if config_value: self.apply_enhancement(domains, enhancement_name, option) enhanced = True diff --git a/certbot/certbot/_internal/display/enhancements.py b/certbot/certbot/_internal/display/enhancements.py deleted file mode 100644 index ce6470708..000000000 --- a/certbot/certbot/_internal/display/enhancements.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Certbot Enhancement Display""" -import logging - -import zope.component - -from certbot import errors -from certbot import interfaces -from certbot.display import util as display_util - -logger = logging.getLogger(__name__) - -# Define a helper function to avoid verbose code -util = zope.component.getUtility - - -def ask(enhancement): - """Display the enhancement to the user. - - :param str enhancement: One of the - :const:`~certbot.plugins.enhancements.ENHANCEMENTS` enhancements - - :returns: True if feature is desired, False otherwise - :rtype: bool - - :raises .errors.Error: if the enhancement provided is not supported - - """ - try: - # Call the appropriate function based on the enhancement - return DISPATCH[enhancement]() - except KeyError: - logger.error("Unsupported enhancement given to ask(): %s", enhancement) - raise errors.Error("Unsupported Enhancement") - - -def redirect_by_default(): - """Determines whether the user would like to redirect to HTTPS. - - :returns: True if redirect is desired, False otherwise - :rtype: bool - - """ - choices = [ - ("No redirect", "Make no further changes to the webserver configuration."), - ("Redirect", "Make all requests redirect to secure HTTPS access. " - "Choose this for new sites, or if you're confident your site works on HTTPS. " - "You can undo this change by editing your web server's configuration."), - ] - - code, selection = util(interfaces.IDisplay).menu( - "Please choose whether or not to redirect HTTP traffic to HTTPS, removing HTTP access.", - choices, default=1, - cli_flag="--redirect / --no-redirect", force_interactive=True) - - if code != display_util.OK: - return False - - return selection == 1 - - -DISPATCH = { - "redirect": redirect_by_default -} diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 4a57dd78d..94ccb8b5e 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -927,7 +927,7 @@ def enhance(config, plugins): config.chain_path = lineage.chain_path if oldstyle_enh: le_client = _init_le_client(config, authenticator=None, installer=installer) - le_client.enhance_config(domains, config.chain_path, ask_redirect=False) + le_client.enhance_config(domains, config.chain_path, redirect_default=False) if enhancements.are_requested(config): enhancements.enable(lineage, domains, installer, config) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 7232ed84b..3e0e5b212 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -577,8 +577,7 @@ class EnhanceConfigTest(ClientTestCommon): self.assertRaises( errors.Error, self.client.enhance_config, [self.domain], None) - @mock.patch("certbot._internal.client.enhancements") - def test_unsupported(self, mock_enhancements): + def test_unsupported(self): self.client.installer = mock.MagicMock() self.client.installer.supported_enhancements.return_value = [] @@ -588,7 +587,6 @@ class EnhanceConfigTest(ClientTestCommon): self.client.enhance_config([self.domain], None) self.assertEqual(mock_logger.warning.call_count, 1) self.client.installer.enhance.assert_not_called() - mock_enhancements.ask.assert_not_called() @mock.patch("certbot._internal.client.logger") def test_already_exists_header(self, mock_log): @@ -612,14 +610,11 @@ class EnhanceConfigTest(ClientTestCommon): self._test_with_already_existing() self.assertFalse(mock_log.warning.called) - @mock.patch("certbot._internal.client.enhancements.ask") @mock.patch("certbot._internal.client.logger") - def test_warn_redirect(self, mock_log, mock_ask): + def test_no_warn_redirect(self, mock_log): self.config.redirect = None - mock_ask.return_value = False - self._test_with_already_existing() - self.assertTrue(mock_log.warning.called) - self.assertTrue("disable" in mock_log.warning.call_args[0][0]) + self._test_with_all_supported() + self.assertFalse(mock_log.warning.called) def test_no_ask_hsts(self): self.config.hsts = True @@ -670,12 +665,6 @@ class EnhanceConfigTest(ClientTestCommon): self.client.installer = installer self._test_error_with_rollback() - @mock.patch("certbot._internal.client.enhancements.ask") - def test_ask(self, mock_ask): - self.config.redirect = None - mock_ask.return_value = True - self._test_with_all_supported() - def _test_error_with_rollback(self): self._test_error() self.assertTrue(self.client.installer.restart.called) diff --git a/certbot/tests/display/enhancements_test.py b/certbot/tests/display/enhancements_test.py deleted file mode 100644 index edace29b1..000000000 --- a/certbot/tests/display/enhancements_test.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Module for enhancement UI.""" -import logging -import unittest - -import mock - -from certbot import errors -from certbot.display import util as display_util - - -class AskTest(unittest.TestCase): - """Test the ask method.""" - def setUp(self): - logging.disable(logging.CRITICAL) - - def tearDown(self): - logging.disable(logging.NOTSET) - - @classmethod - def _call(cls, enhancement): - from certbot._internal.display.enhancements import ask - return ask(enhancement) - - @mock.patch("certbot._internal.display.enhancements.util") - def test_redirect(self, mock_util): - mock_util().menu.return_value = (display_util.OK, 1) - self.assertTrue(self._call("redirect")) - - def test_key_error(self): - self.assertRaises(errors.Error, self._call, "unknown_enhancement") - - -class RedirectTest(unittest.TestCase): - """Test the redirect_by_default method.""" - @classmethod - def _call(cls): - from certbot._internal.display.enhancements import redirect_by_default - return redirect_by_default() - - @mock.patch("certbot._internal.display.enhancements.util") - def test_secure(self, mock_util): - mock_util().menu.return_value = (display_util.OK, 1) - self.assertTrue(self._call()) - - @mock.patch("certbot._internal.display.enhancements.util") - def test_cancel(self, mock_util): - mock_util().menu.return_value = (display_util.CANCEL, 1) - self.assertFalse(self._call()) - - @mock.patch("certbot._internal.display.enhancements.util") - def test_easy(self, mock_util): - mock_util().menu.return_value = (display_util.OK, 0) - self.assertFalse(self._call()) - - -if __name__ == "__main__": - unittest.main() # pragma: no cover From bc3088121b74158ca4b24bc380d043f187196957 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 31 Mar 2020 22:12:42 +0200 Subject: [PATCH 66/92] Add a step to check powershell version in vs2017-win2016 (#7870) Following discussion at https://github.com/certbot/certbot/pull/7539#issuecomment-572318805, this PR adds a check for Powershell version: we expect that the `vs2017-win2016` node that will test the installer has Powershell 5.x, and nothing else. This ensure that at least one node of the pipeline is testing the installer with the lowest Powershell version supported by Certbot. One full pipeline success can be seen here: https://dev.azure.com/adferrand/certbot/_build/results?buildId=713 I also create on purpose a failing pipeline, that would check that Powershell 6.x is installed. Its result can be seen here: https://dev.azure.com/adferrand/certbot/_build/results?buildId=714 --- .azure-pipelines/templates/installer-tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.azure-pipelines/templates/installer-tests.yml b/.azure-pipelines/templates/installer-tests.yml index ea2101792..ebadcb2dc 100644 --- a/.azure-pipelines/templates/installer-tests.yml +++ b/.azure-pipelines/templates/installer-tests.yml @@ -31,6 +31,13 @@ jobs: pool: vmImage: $(imageName) steps: + - powershell: | + $currentVersion = $PSVersionTable.PSVersion + if ($currentVersion.Major -ne 5) { + throw "Powershell version is not 5.x" + } + condition: eq(variables['imageName'], 'vs2017-win2016') + displayName: Check Powershell 5.x is used in vs2017-win2016 - task: UsePythonVersion@0 inputs: versionSpec: 3.8 From 4ca86d9482cbb9bca04335bfb7dbd892cefe91c8 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Thu, 2 Apr 2020 08:53:58 +1100 Subject: [PATCH 67/92] acme: socket timeout for HTTP standalone servers (#7388) * acme: socket timeout for HTTP standalone servers Adds a default 30 second timeout to the StreamRequestHandler for clients connecting to standalone HTTP-01 servers. This should prevent most cases of an idle client connection from preventing the standalone server from shutting down. Fixes #7386 * use idiomatic kwargs default value * move HTTP01Server lower to fix mypy forward ref. * fix test crash on macOS due to socket double-close * maybe its not an OSError? * disable coverage check on useless branch --- acme/acme/standalone.py | 10 ++++++---- acme/tests/standalone_test.py | 22 ++++++++++++++++++++++ certbot/CHANGELOG.md | 2 ++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 52ac07915..7a61ba868 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -176,10 +176,10 @@ class HTTPServer(BaseHTTPServer.HTTPServer): class HTTP01Server(HTTPServer, ACMEServerMixin): """HTTP01 Server.""" - def __init__(self, server_address, resources, ipv6=False): + def __init__(self, server_address, resources, ipv6=False, timeout=30): HTTPServer.__init__( self, server_address, HTTP01RequestHandler.partial_init( - simple_http_resources=resources), ipv6=ipv6) + simple_http_resources=resources, timeout=timeout), ipv6=ipv6) class HTTP01DualNetworkedServers(BaseDualNetworkedServers): @@ -204,6 +204,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.simple_http_resources = kwargs.pop("simple_http_resources", set()) + self.timeout = kwargs.pop('timeout', 30) BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def log_message(self, format, *args): # pylint: disable=redefined-builtin @@ -253,7 +254,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.path) @classmethod - def partial_init(cls, simple_http_resources): + def partial_init(cls, simple_http_resources, timeout): """Partially initialize this handler. This is useful because `socketserver.BaseServer` takes @@ -262,7 +263,8 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """ return functools.partial( - cls, simple_http_resources=simple_http_resources) + cls, simple_http_resources=simple_http_resources, + timeout=timeout) class _BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler): diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 8c08ab89b..8face7c7b 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -85,6 +85,28 @@ class HTTP01ServerTest(unittest.TestCase): def test_http01_not_found(self): self.assertFalse(self._test_http01(add=False)) + def test_timely_shutdown(self): + from acme.standalone import HTTP01Server + server = HTTP01Server(('', 0), resources=set(), timeout=0.05) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.start() + + client = socket.socket() + client.connect(('localhost', server.socket.getsockname()[1])) + + stop_thread = threading.Thread(target=server.shutdown) + stop_thread.start() + server_thread.join(5.) + + is_hung = server_thread.is_alive() + try: + client.shutdown(socket.SHUT_RDWR) + except: # pragma: no cover, pylint: disable=bare-except + # may raise error because socket could already be closed + pass + + self.assertFalse(is_hung, msg='Server shutdown should not be hung') + @unittest.skipIf(not challenges.TLSALPN01.is_supported(), "pyOpenSSL too old") class TLSALPN01ServerTest(unittest.TestCase): diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 88a4ab190..99c1de63b 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -26,6 +26,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Fix nginx plugin crash when non-ASCII configuration file is being read (instead, the user will be warned that UTF-8 must be used). * Fix hanging OCSP queries during revocation checking - added a 10 second timeout. +* Standalone servers now have a default socket timeout of 30 seconds, fixing + cases where an idle connection can cause the standalone plugin to hang. More details about these changes can be found on our GitHub repo. From 5992d521e225fd3abf176f07bdc7753a9568526b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 7 Apr 2020 15:26:19 -0700 Subject: [PATCH 68/92] Print boulder logs when boulder setup fails (#7885) This is part of https://github.com/certbot/certbot/issues/7303. * Print boulder logs if boulder fails to start * Print description and fix command. * Change output to stderr. --- .../utils/acme_server.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 4b20d5acc..fb202005e 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -38,7 +38,7 @@ class ACMEServer(object): :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) :param list nodes: list of node names that will be setup by pytest xdist :param bool http_proxy: if False do not start the HTTP proxy - :param bool stdout: if True stream subprocesses stdout to standard stdout + :param bool stdout: if True stream all subprocesses stdout to standard stdout """ self._construct_acme_xdist(acme_server, nodes) @@ -165,17 +165,24 @@ class ACMEServer(object): os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'), join(instance_path, 'test/rate-limit-policies.yml')) - # Launch the Boulder server - self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path) + try: + # Launch the Boulder server + self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path) - # Wait for the ACME CA server to be up. - print('=> Waiting for boulder instance to respond...') - misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300) + # Wait for the ACME CA server to be up. + print('=> Waiting for boulder instance to respond...') + misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300) - # Configure challtestsrv to answer any A record request with ip of the docker host. - response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), - json={'ip': '10.77.77.1'}) - response.raise_for_status() + # Configure challtestsrv to answer any A record request with ip of the docker host. + response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), + json={'ip': '10.77.77.1'}) + response.raise_for_status() + except BaseException: + # If we failed to set up boulder, print its logs. + print('=> Boulder setup failed. Boulder logs are:') + process = self._launch_process(['docker-compose', 'logs'], cwd=instance_path, force_stderr=True) + process.wait() + raise print('=> Finished boulder instance deployment.') @@ -188,11 +195,12 @@ class ACMEServer(object): self._launch_process(command) print('=> Finished configuring the HTTP proxy.') - def _launch_process(self, command, cwd=os.getcwd(), env=None): + def _launch_process(self, command, cwd=os.getcwd(), env=None, force_stderr=False): """Launch silently a subprocess OS command""" if not env: env = os.environ - process = subprocess.Popen(command, stdout=self._stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env) + stdout = sys.stderr if force_stderr else self._stdout + process = subprocess.Popen(command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env) self._processes.append(process) return process From e9895d2ec69395cfcf1cfff35e7668b57e1da00f Mon Sep 17 00:00:00 2001 From: alexzorin Date: Wed, 8 Apr 2020 09:19:13 +1000 Subject: [PATCH 69/92] Fix fullchain parsing for CRLF chains (#7877) Fixes #7875 . After [this comment](https://github.com/certbot/certbot/issues/7875#issuecomment-608145208) and evaluating the options, I opted to go with `stricttextualmsg`, as required by RFC 8555. Reasoning is that the ACME v1 code path (via OpenSSL) produces a `fullchain_pem` which satisfies `stricttextualmsg`, so we don't need to be more generous than that. One downside of the `re` approach is that it doesn't seem capable of capturing repeating group matches. As a result, it matches each certificate individually, silently passing over any data in between the encapsulation boundaries, such as explanatory text, which is prohibited by RFC 8555. It would be ideal to raise an error when encountering such a non-conformant chain, but we'd need to create a mini-parser to do it, I think. * Fix fullchain parsing for CRLF chains. fullchain parsing now works in two passes: 1. A first pass which is generous with what it accepts - basically preeb(CERTIFICATE)+anything+posteb(CERTIFICATE). This determines the boundaries for each certificate. 2. A second pass which normalizes (by parsing and re-encoding) each certificate found in the first pass. * typo in docstring * remove redundant group in regex * can't use assertRaisesRegex until py27 is gone --- certbot/CHANGELOG.md | 2 ++ certbot/certbot/crypto_util.py | 33 +++++++++++++++++++++++++++---- certbot/tests/crypto_util_test.py | 18 ++++++++++++++++- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 99c1de63b..7813c4db3 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -28,6 +28,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Fix hanging OCSP queries during revocation checking - added a 10 second timeout. * Standalone servers now have a default socket timeout of 30 seconds, fixing cases where an idle connection can cause the standalone plugin to hang. +* Parsing of the RFC 8555 application/pem-certificate-chain now tolerates CRLF line + endings. This should fix interoperability with Buypass' services. More details about these changes can be found on our GitHub repo. diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index adb972f24..168531667 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -8,6 +8,7 @@ import hashlib import logging import warnings +import re # See https://github.com/pyca/cryptography/issues/4275 from cryptography import x509 # type: ignore from cryptography.exceptions import InvalidSignature @@ -478,6 +479,17 @@ def sha256sum(filename): sha256.update(file_d.read().encode('UTF-8')) return sha256.hexdigest() +# Finds one CERTIFICATE stricttextualmsg according to rfc7468#section-3. +# Does not validate the base64text - use crypto.load_certificate. +CERT_PEM_REGEX = re.compile( + b"""-----BEGIN CERTIFICATE-----\r? +.+?\r? +-----END CERTIFICATE-----\r? +""", + re.DOTALL # DOTALL (/s) because the base64text may include newlines +) + + def cert_and_chain_from_fullchain(fullchain_pem): """Split fullchain_pem into cert_pem and chain_pem @@ -486,11 +498,24 @@ def cert_and_chain_from_fullchain(fullchain_pem): :returns: tuple of string cert_pem and chain_pem :rtype: tuple + :raises errors.Error: If there are less than 2 certificates in the chain. + """ - cert = crypto.dump_certificate(crypto.FILETYPE_PEM, - crypto.load_certificate(crypto.FILETYPE_PEM, fullchain_pem)).decode() - chain = fullchain_pem[len(cert):].lstrip() - return (cert, chain) + # First pass: find the boundary of each certificate in the chain. + # TODO: This will silently skip over any "explanatory text" in between boundaries, + # which is prohibited by RFC8555. + certs = CERT_PEM_REGEX.findall(fullchain_pem.encode()) + if len(certs) < 2: + raise errors.Error("failed to parse fullchain into cert and chain: " + + "less than 2 certificates in chain") + + # Second pass: for each certificate found, parse it using OpenSSL and re-encode it, + # with the effect of normalizing any encoding variations (e.g. CRLF, whitespace). + certs_normalized = [crypto.dump_certificate(crypto.FILETYPE_PEM, + crypto.load_certificate(crypto.FILETYPE_PEM, cert)).decode() for cert in certs] + + # Since each normalized cert has a newline suffix, no extra newlines are required. + return (certs_normalized[0], "".join(certs_normalized[1:])) def get_serial_from_cert(cert_path): """Retrieve the serial number of a certificate from certificate path diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 1d642ae9e..d52e3acdb 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -376,17 +376,33 @@ class Sha256sumTest(unittest.TestCase): class CertAndChainFromFullchainTest(unittest.TestCase): """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" + def _parse_and_reencode_pem(self, cert_pem): + from OpenSSL import crypto + return crypto.dump_certificate(crypto.FILETYPE_PEM, + crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)).decode() + def test_cert_and_chain_from_fullchain(self): cert_pem = CERT.decode() chain_pem = cert_pem + SS_CERT.decode() fullchain_pem = cert_pem + chain_pem spacey_fullchain_pem = cert_pem + u'\n' + chain_pem + crlf_fullchain_pem = fullchain_pem.replace(u'\n', u'\r\n') + + # In the ACME v1 code path, the fullchain is constructed by loading cert+chain DERs + # and using OpenSSL to dump them, so here we confirm that OpenSSL is producing certs + # that will be parseable by cert_and_chain_from_fullchain. + acmev1_fullchain_pem = self._parse_and_reencode_pem(cert_pem) + \ + self._parse_and_reencode_pem(cert_pem) + self._parse_and_reencode_pem(SS_CERT.decode()) + from certbot.crypto_util import cert_and_chain_from_fullchain - for fullchain in (fullchain_pem, spacey_fullchain_pem): + for fullchain in (fullchain_pem, spacey_fullchain_pem, crlf_fullchain_pem, + acmev1_fullchain_pem): cert_out, chain_out = cert_and_chain_from_fullchain(fullchain) self.assertEqual(cert_out, cert_pem) self.assertEqual(chain_out, chain_pem) + self.assertRaises(errors.Error, cert_and_chain_from_fullchain, cert_pem) + if __name__ == '__main__': unittest.main() # pragma: no cover From 537bee09940abe064fe7d5daff25b7bb3748867c Mon Sep 17 00:00:00 2001 From: inejge Date: Thu, 9 Apr 2020 20:25:39 +0200 Subject: [PATCH 70/92] Add minimal proxy support for OCSP verification (#7892) Translate a proxy specified by an environment variable ("http_proxy" or "HTTP_PROXY") into options recognized by "openssl ocsp". Support is limited to HTTP proxies which don't require authentication. Fixes #6150 --- AUTHORS.md | 1 + certbot/CHANGELOG.md | 1 + certbot/certbot/ocsp.py | 20 ++++++++++++++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index f5b981b8e..4414076fc 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -103,6 +103,7 @@ Authors * [Henry Chen](https://github.com/henrychen95) * [Hugo van Kemenade](https://github.com/hugovk) * [Ingolf Becker](https://github.com/watercrossing) +* [Ivan Nejgebauer](https://github.com/inejge) * [Jaap Eldering](https://github.com/eldering) * [Jacob Hoffman-Andrews](https://github.com/jsha) * [Jacob Sachs](https://github.com/jsachs) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 7813c4db3..f61cdcfc7 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -14,6 +14,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). of all domains challenged for the current certificate. * Added TLS-ALPN-01 challenge support in the `acme` library. Support of this challenge in the Certbot client is planned to be added in a future release. +* Added minimal proxy support for OCSP verification. ### Changed diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 1d5611b64..863c5f163 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -21,6 +21,7 @@ from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import util +from certbot.compat.os import getenv from certbot.interfaces import RenewableCert # pylint: disable=unused-import try: @@ -102,17 +103,32 @@ class RevocationChecker(object): def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url, timeout): # type: (str, str, str, str, int) -> bool + # Minimal implementation of proxy selection logic as seen in, e.g., cURL + # Some things that won't work, but may well be in use somewhere: + # - username and password for proxy authentication + # - proxies accepting TLS connections + # - proxy exclusion through NO_PROXY + env_http_proxy = getenv('http_proxy') + env_HTTP_PROXY = getenv('HTTP_PROXY') + proxy_host = None + if env_http_proxy is not None or env_HTTP_PROXY is not None: + proxy_host = env_http_proxy if env_http_proxy is not None else env_HTTP_PROXY + if proxy_host is None: + url_opts = ["-url", url] + else: + if proxy_host.startswith('http://'): + proxy_host = proxy_host[len('http://'):] + url_opts = ["-host", proxy_host, "-path", url] # jdkasten thanks "Bulletproof SSL and TLS - Ivan Ristic" for documenting this! cmd = ["openssl", "ocsp", "-no_nonce", "-issuer", chain_path, "-cert", cert_path, - "-url", url, "-CAfile", chain_path, "-verify_other", chain_path, "-trust_other", "-timeout", str(timeout), - "-header"] + self.host_args(host) + "-header"] + self.host_args(host) + url_opts logger.debug("Querying OCSP for %s", cert_path) logger.debug(" ".join(cmd)) try: From 316e4640f86c7f48a48d194a77e9462d1f50bd34 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 9 Apr 2020 14:35:47 -0700 Subject: [PATCH 71/92] Upgrade the test farm tests to use Python 3 (#7876) Fixes #7857. * stop using urllib2 in test farm tests * use six for urllib instead * remove fabric lcd usage * correct lcd removal * remove fabric cd * convert some remote calls to v2 * move more cxns to v2 * get run working with prefix * get sudo commands working * remove final fabric v1 references including local * update requirements and README * add new venv to gitignore * update version used in travis * remove deploy_script unused kwargs * fix killboulder implementation so I can test creating a new boulder server * hardcode the gopath due to broken env manamagement in fabric2 * Update letstest readme * move the comment about hardcoding the ggopath * catch BaseException instead of Exception * work around fabric #2007 * use connections as context managers to ensure they're closed * remove reference to virtualenv --- .gitignore | 1 + .travis.yml | 8 +- tests/letstest/README.md | 10 +- tests/letstest/multitester.py | 235 ++++++++++++++++---------------- tests/letstest/requirements.txt | 4 +- 5 files changed, 133 insertions(+), 125 deletions(-) diff --git a/.gitignore b/.gitignore index 6dd422187..6505e716c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ tags tests/letstest/letest-*/ tests/letstest/*.pem tests/letstest/venv/ +tests/letstest/venv3/ .venv diff --git a/.travis.yml b/.travis.yml index d498d0305..ae25a6895 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,24 +90,24 @@ matrix: before_install: addons: <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-apache2 - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-leauto-upgrades - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" git: depth: false # This is needed to have the history to checkout old versions of certbot-auto. <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-certonly-standalone - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-sdists - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" diff --git a/tests/letstest/README.md b/tests/letstest/README.md index f8a15208e..5bd326e2a 100644 --- a/tests/letstest/README.md +++ b/tests/letstest/README.md @@ -15,12 +15,12 @@ Simple AWS testfarm scripts for certbot client testing are needed, they need to be requested via online webform. ## Installation and configuration -These tests require Python 2.7, awscli, boto3, PyYAML, and fabric<2.0. If you -have Python 2.7 and virtualenv installed, you can use requirements.txt to -create a virtual environment with a known set of dependencies by running: +These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you +have Python 3 installed, you can use requirements.txt to create a virtual +environment with a known set of dependencies by running: ``` -virtualenv --python $(command -v python2.7 || command -v python2 || command -v python) venv -. ./venv/bin/activate +python3 -m venv venv3 +. ./venv3/bin/activate pip install --requirement requirements.txt ``` diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 9ea9fe76b..09821e7dd 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -40,23 +40,16 @@ import socket import sys import time import traceback -import urllib2 import boto3 from botocore.exceptions import ClientError +from six.moves.urllib import error as urllib_error +from six.moves.urllib import request as urllib_request import yaml -import fabric -from fabric.api import cd -from fabric.api import env -from fabric.api import execute -from fabric.api import lcd -from fabric.api import local -from fabric.api import run -from fabric.api import sudo -from fabric.context_managers import shell_env -from fabric.operations import get -from fabric.operations import put +from fabric import Config +from fabric import Connection + # Command line parser #------------------------------------------------------------------------------- @@ -203,11 +196,11 @@ def block_until_http_ready(urlstring, wait_time=10, timeout=240): try: sys.stdout.write('.') sys.stdout.flush() - req = urllib2.Request(urlstring) - response = urllib2.urlopen(req) + req = urllib_request.Request(urlstring) + response = urllib_request.urlopen(req) #if response.code == 200: server_ready = True - except urllib2.URLError: + except urllib_error.URLError: pass time.sleep(wait_time) t_elapsed += wait_time @@ -244,76 +237,85 @@ def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20 # Fabric Routines #------------------------------------------------------------------------------- -def local_git_clone(repo_url): +def local_git_clone(local_cxn, repo_url): "clones master of repo_url" - with lcd(LOGDIR): - local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s letsencrypt'% repo_url) - local('tar czf le.tar.gz letsencrypt') + local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR) + local_cxn.local('cd %s && git clone %s letsencrypt'% (LOGDIR, repo_url)) + local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt'% LOGDIR) -def local_git_branch(repo_url, branch_name): +def local_git_branch(local_cxn, repo_url, branch_name): "clones branch of repo_url" - with lcd(LOGDIR): - local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s letsencrypt --branch %s --single-branch'%(repo_url, branch_name)) - local('tar czf le.tar.gz letsencrypt') + local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR) + local_cxn.local('cd %s && git clone %s letsencrypt --branch %s --single-branch'% + (LOGDIR, repo_url, branch_name)) + local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt' % LOGDIR) -def local_git_PR(repo_url, PRnumstr, merge_master=True): +def local_git_PR(local_cxn, repo_url, PRnumstr, merge_master=True): "clones specified pull request from repo_url and optionally merges into master" - with lcd(LOGDIR): - local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s letsencrypt'% repo_url) - local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr) - local('cd letsencrypt && git checkout lePRtest') - if merge_master: - local('cd letsencrypt && git remote update origin') - local('cd letsencrypt && git merge origin/master -m "testmerge"') - local('tar czf le.tar.gz letsencrypt') + local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR) + local_cxn.local('cd %s && git clone %s letsencrypt' % (LOGDIR, repo_url)) + local_cxn.local('cd %s && cd letsencrypt && ' + 'git fetch origin pull/%s/head:lePRtest' % (LOGDIR, PRnumstr)) + local_cxn.local('cd %s && cd letsencrypt && git checkout lePRtest' % LOGDIR) + if merge_master: + local_cxn.local('cd %s && cd letsencrypt && git remote update origin' % LOGDIR) + local_cxn.local('cd %s && cd letsencrypt && ' + 'git merge origin/master -m "testmerge"' % LOGDIR) + local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt' % LOGDIR) -def local_repo_to_remote(): +def local_repo_to_remote(cxn): "copies local tarball of repo to remote" - with lcd(LOGDIR): - put(local_path='le.tar.gz', remote_path='') - run('tar xzf le.tar.gz') + filename = 'le.tar.gz' + local_path = os.path.join(LOGDIR, filename) + cxn.put(local=local_path, remote='') + cxn.run('tar xzf %s' % filename) -def local_repo_clean(): +def local_repo_clean(local_cxn): "delete tarball" - with lcd(LOGDIR): - local('rm le.tar.gz') + filename = 'le.tar.gz' + local_path = os.path.join(LOGDIR, filename) + local_cxn.local('rm %s' % local_path) -def deploy_script(scriptpath, *args): +def deploy_script(cxn, scriptpath, *args): "copies to remote and executes local script" - #with lcd('scripts'): - put(local_path=scriptpath, remote_path='', mirror_local_mode=True) + cxn.put(local=scriptpath, remote='', preserve_mode=True) scriptfile = os.path.split(scriptpath)[1] args_str = ' '.join(args) - run('./'+scriptfile+' '+args_str) + cxn.run('./'+scriptfile+' '+args_str) -def run_boulder(): - with cd('$GOPATH/src/github.com/letsencrypt/boulder'): - run('sudo docker-compose up -d') +def run_boulder(cxn): + boulder_path = '$GOPATH/src/github.com/letsencrypt/boulder' + cxn.run('cd %s && sudo docker-compose up -d' % boulder_path) -def config_and_launch_boulder(instance): - execute(deploy_script, 'scripts/boulder_config.sh') - execute(run_boulder) +def config_and_launch_boulder(cxn, instance): + # yes, we're hardcoding the gopath. it's a predetermined AMI. + with cxn.prefix('export GOPATH=/home/ubuntu/gopath'): + deploy_script(cxn, 'scripts/boulder_config.sh') + run_boulder(cxn) -def install_and_launch_certbot(instance, boulder_url, target): - execute(local_repo_to_remote) - with shell_env(BOULDER_URL=boulder_url, - PUBLIC_IP=instance.public_ip_address, - PRIVATE_IP=instance.private_ip_address, - PUBLIC_HOSTNAME=instance.public_dns_name, - PIP_EXTRA_INDEX_URL=cl_args.alt_pip, - OS_TYPE=target['type']): - execute(deploy_script, cl_args.test_script) +def install_and_launch_certbot(cxn, instance, boulder_url, target): + local_repo_to_remote(cxn) + # This needs to be like this, I promise. 1) The env argument to run doesn't work. + # See https://github.com/fabric/fabric/issues/1744. 2) prefix() sticks an && between + # the commands, so it needs to be exports rather than no &&s in between for the script subshell. + with cxn.prefix('export BOULDER_URL=%s && export PUBLIC_IP=%s && export PRIVATE_IP=%s && ' + 'export PUBLIC_HOSTNAME=%s && export PIP_EXTRA_INDEX_URL=%s && ' + 'export OS_TYPE=%s' % + (boulder_url, + instance.public_ip_address, + instance.private_ip_address, + instance.public_dns_name, + cl_args.alt_pip, + target['type'])): + deploy_script(cxn, cl_args.test_script) -def grab_certbot_log(): +def grab_certbot_log(cxn): "grabs letsencrypt.log via cat into logged stdout" - sudo('if [ -f /var/log/letsencrypt/letsencrypt.log ]; then \ - cat /var/log/letsencrypt/letsencrypt.log; else echo "[novarlog]"; fi') + cxn.sudo('/bin/bash -l -i -c \'if [ -f "/var/log/letsencrypt/letsencrypt.log" ]; then ' + + 'cat "/var/log/letsencrypt/letsencrypt.log"; else echo "[novarlog]"; fi\'') # fallback file if /var/log is unwriteable...? correct? - sudo('if [ -f ./certbot.log ]; then \ - cat ./certbot.log; else echo "[nolocallog]"; fi') + cxn.sudo('/bin/bash -l -i -c \'if [ -f ./certbot.log ]; then ' + + 'cat ./certbot.log; else echo "[nolocallog]"; fi\'') def create_client_instance(ec2_client, target, security_group_id, subnet_id): @@ -341,7 +343,7 @@ def create_client_instance(ec2_client, target, security_group_id, subnet_id): userdata=userdata) -def test_client_process(inqueue, outqueue, boulder_url): +def test_client_process(fab_config, inqueue, outqueue, boulder_url): cur_proc = mp.current_process() for inreq in iter(inqueue.get, SENTINEL): ii, instance_id, target = inreq @@ -358,30 +360,31 @@ def test_client_process(inqueue, outqueue, boulder_url): print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name'])) instance = block_until_instance_ready(instance) print("server %s at %s"%(instance, instance.public_ip_address)) - env.host_string = "%s@%s"%(target['user'], instance.public_ip_address) - print(env.host_string) + host_string = "%s@%s"%(target['user'], instance.public_ip_address) + print(host_string) - try: - install_and_launch_certbot(instance, boulder_url, target) - outqueue.put((ii, target, Status.PASS)) - print("%s - %s SUCCESS"%(target['ami'], target['name'])) - except: - outqueue.put((ii, target, Status.FAIL)) - print("%s - %s FAIL"%(target['ami'], target['name'])) - traceback.print_exc(file=sys.stdout) - pass + with Connection(host_string, config=fab_config) as cxn: + try: + install_and_launch_certbot(cxn, instance, boulder_url, target) + outqueue.put((ii, target, Status.PASS)) + print("%s - %s SUCCESS"%(target['ami'], target['name'])) + except: + outqueue.put((ii, target, Status.FAIL)) + print("%s - %s FAIL"%(target['ami'], target['name'])) + traceback.print_exc(file=sys.stdout) + pass - # append server certbot.log to each per-machine output log - print("\n\ncertbot.log\n" + "-"*80 + "\n") - try: - execute(grab_certbot_log) - except: - print("log fail\n") - traceback.print_exc(file=sys.stdout) - pass + # append server certbot.log to each per-machine output log + print("\n\ncertbot.log\n" + "-"*80 + "\n") + try: + grab_certbot_log(cxn) + except: + print("log fail\n") + traceback.print_exc(file=sys.stdout) + pass -def cleanup(cl_args, instances, targetlist): +def cleanup(cl_args, instances, targetlist, boulder_server): print('Logs in ', LOGDIR) # If lengths of instances and targetlist aren't equal, instances failed to # start before running tests so leaving instances running for debugging @@ -402,19 +405,25 @@ def cleanup(cl_args, instances, targetlist): def main(): # Fabric library controlled through global env parameters - env.key_filename = KEYFILE - env.shell = '/bin/bash -l -i -c' - env.connection_attempts = 5 - env.timeout = 10 - # replace default SystemExit thrown by fabric during trouble - class FabricException(Exception): - pass - env['abort_exception'] = FabricException + fab_config = Config(overrides={ + "connect_kwargs": { + "key_filename": [KEYFILE], # https://github.com/fabric/fabric/issues/2007 + }, + "run": { + "echo": True, + "pty": True, + }, + "timeouts": { + "connect": 10, + }, + }) + # no network connection, so don't worry about closing this one. + local_cxn = Connection('localhost', config=fab_config) # Set up local copy of git repo #------------------------------------------------------------------------------- print("Making local dir for test repo and logs: %s"%LOGDIR) - local('mkdir %s'%LOGDIR) + local_cxn.local('mkdir %s'%LOGDIR) # figure out what git object to test and locally create it in LOGDIR print("Making local git repo") @@ -422,14 +431,14 @@ def main(): if cl_args.pull_request != '~': print('Testing PR %s '%cl_args.pull_request, "MERGING into master" if cl_args.merge_master else "") - execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master) + local_git_PR(local_cxn, cl_args.repo, cl_args.pull_request, cl_args.merge_master) elif cl_args.branch != '~': print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo)) - execute(local_git_branch, cl_args.repo, cl_args.branch) + local_git_branch(local_cxn, cl_args.repo, cl_args.branch) else: print('Testing master of %s'%cl_args.repo) - execute(local_git_clone, cl_args.repo) - except FabricException: + local_git_clone(local_cxn, cl_args.repo) + except BaseException: print("FAIL: trouble with git repo") traceback.print_exc() exit() @@ -437,7 +446,7 @@ def main(): # Set up EC2 instances #------------------------------------------------------------------------------- - configdata = yaml.load(open(cl_args.config_file, 'r')) + configdata = yaml.safe_load(open(cl_args.config_file, 'r')) targetlist = configdata['targets'] print('Testing against these images: [%d total]'%len(targetlist)) for target in targetlist: @@ -511,15 +520,16 @@ def main(): print(" server %s"%boulder_server) - # env.host_string defines the ssh user and host for connection - env.host_string = "ubuntu@%s"%boulder_server.public_ip_address - print("Boulder Server at (SSH):", env.host_string) + # host_string defines the ssh user and host for connection + host_string = "ubuntu@%s"%boulder_server.public_ip_address + print("Boulder Server at (SSH):", host_string) if not boulder_preexists: print("Configuring and Launching Boulder") - config_and_launch_boulder(boulder_server) - # blocking often unnecessary, but cheap EC2 VMs can get very slow - block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, - wait_time=10, timeout=500) + with Connection(host_string, config=fab_config) as boulder_cxn: + config_and_launch_boulder(boulder_cxn, boulder_server) + # blocking often unnecessary, but cheap EC2 VMs can get very slow + block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, + wait_time=10, timeout=500) boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) @@ -545,7 +555,7 @@ def main(): # initiate process execution for i in range(num_processes): - p = mp.Process(target=test_client_process, args=(inqueue, outqueue, boulder_url)) + p = mp.Process(target=test_client_process, args=(fab_config, inqueue, outqueue, boulder_url)) jobs.append(p) p.daemon = True # kills subprocesses if parent is killed p.start() @@ -569,7 +579,7 @@ def main(): outqueue.put(SENTINEL) # clean up - execute(local_repo_clean) + local_repo_clean(local_cxn) # print and save summary results results_file = open(LOGDIR+'/results', 'w') @@ -594,10 +604,7 @@ def main(): sys.exit(1) finally: - cleanup(cl_args, instances, targetlist) - - # kill any connections - fabric.network.disconnect_all() + cleanup(cl_args, instances, targetlist, boulder_server) if __name__ == '__main__': diff --git a/tests/letstest/requirements.txt b/tests/letstest/requirements.txt index 24bd77331..840e3e5d5 100644 --- a/tests/letstest/requirements.txt +++ b/tests/letstest/requirements.txt @@ -5,9 +5,9 @@ cffi==1.14.0 cryptography==2.8 docutils==0.15.2 enum34==1.1.9 -Fabric==1.14.1 -futures==3.3.0 +Fabric==2.5.0 ipaddress==1.0.23 +Invoke==1.4.1 jmespath==0.9.5 paramiko==2.7.1 pycparser==2.19 From 8e4dc0a48c6108311cf68ca1c683d29b15d49c97 Mon Sep 17 00:00:00 2001 From: Karan Suthar Date: Mon, 13 Apr 2020 23:11:39 +0530 Subject: [PATCH 72/92] Minor bugfixes (#7891) * Fix dangerous default argument * Remove unused imports * Remove unnecessary comprehension * Use literal syntax to create data structure * Use literal syntax instead of function calls to create data structure Co-authored-by: deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com> --- acme/acme/client.py | 4 +- acme/docs/conf.py | 1 - acme/tests/errors_test.py | 2 +- .../certbot_apache/_internal/apache_util.py | 4 +- .../certbot_apache/_internal/assertions.py | 4 +- .../certbot_apache/_internal/augeasparser.py | 6 +- .../certbot_apache/_internal/configurator.py | 18 +++--- .../certbot_apache/_internal/display_ops.py | 4 +- .../certbot_apache/_internal/dualparser.py | 4 +- .../certbot_apache/_internal/parser.py | 6 +- .../_internal/parsernode_util.py | 2 +- certbot-apache/tests/augeasnode_test.py | 1 - certbot-apache/tests/centos6_test.py | 6 +- certbot-apache/tests/centos_test.py | 4 +- certbot-apache/tests/configurator_test.py | 42 ++++++------- certbot-apache/tests/display_ops_test.py | 4 +- certbot-apache/tests/gentoo_test.py | 6 +- certbot-apache/tests/obj_test.py | 16 ++--- certbot-apache/tests/util.py | 38 ++++++------ .../certbot_tests/context.py | 1 - .../certbot_integration_tests/utils/misc.py | 2 +- .../certbot_compatibility_test/test_driver.py | 2 +- .../certbot_nginx/_internal/configurator.py | 4 +- .../certbot_nginx/_internal/display_ops.py | 4 +- .../certbot_nginx/_internal/parser.py | 6 +- .../certbot_nginx/_internal/parser_obj.py | 4 +- certbot-nginx/tests/configurator_test.py | 14 ++--- certbot-nginx/tests/obj_test.py | 36 +++++------ certbot-nginx/tests/parser_obj_test.py | 2 +- certbot-nginx/tests/parser_test.py | 62 +++++++++---------- .../certbot/_internal/cli/cli_constants.py | 14 ++--- certbot/certbot/_internal/main.py | 2 +- certbot/certbot/_internal/plugins/manual.py | 2 +- .../certbot/_internal/plugins/selection.py | 2 +- certbot/certbot/display/ops.py | 2 +- certbot/certbot/plugins/storage.py | 4 +- certbot/tests/display/ops_test.py | 8 +-- certbot/tests/display/util_test.py | 10 +-- certbot/tests/error_handler_test.py | 2 +- certbot/tests/hook_test.py | 2 +- certbot/tests/main_test.py | 4 +- certbot/tests/plugins/common_test.py | 12 ++-- certbot/tests/plugins/standalone_test.py | 6 +- certbot/tests/reverter_test.py | 8 +-- .../rebuild_dependencies.py | 2 +- tools/_venv_common.py | 4 +- tools/deactivate.py | 1 - windows-installer/construct.py | 4 +- 48 files changed, 197 insertions(+), 201 deletions(-) mode change 100755 => 100644 letsencrypt-auto-source/rebuild_dependencies.py diff --git a/acme/acme/client.py b/acme/acme/client.py index cbe543f91..3ce321ac9 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1123,8 +1123,8 @@ class ClientNetwork(object): debug_content = response.content.decode("utf-8") logger.debug('Received response:\nHTTP %d\n%s\n\n%s', response.status_code, - "\n".join(["{0}: {1}".format(k, v) - for k, v in response.headers.items()]), + "\n".join("{0}: {1}".format(k, v) + for k, v in response.headers.items()), debug_content) return response diff --git a/acme/docs/conf.py b/acme/docs/conf.py index a9c69d538..ba1a3aa8b 100644 --- a/acme/docs/conf.py +++ b/acme/docs/conf.py @@ -13,7 +13,6 @@ # serve to show the default. import os -import shlex import sys here = os.path.abspath(os.path.dirname(__file__)) diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index 1e5f3d479..c9c6f484f 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -35,7 +35,7 @@ class PollErrorTest(unittest.TestCase): def setUp(self): from acme.errors import PollError self.timeout = PollError( - exhausted=set([mock.sentinel.AR]), + exhausted={mock.sentinel.AR}, updated={}) self.invalid = PollError(exhausted=set(), updated={ mock.sentinel.AR: mock.sentinel.AR2}) diff --git a/certbot-apache/certbot_apache/_internal/apache_util.py b/certbot-apache/certbot_apache/_internal/apache_util.py index ebc8a26c9..862685027 100644 --- a/certbot-apache/certbot_apache/_internal/apache_util.py +++ b/certbot-apache/certbot_apache/_internal/apache_util.py @@ -130,7 +130,7 @@ def included_in_paths(filepath, paths): :rtype: bool """ - return any([fnmatch.fnmatch(filepath, path) for path in paths]) + return any(fnmatch.fnmatch(filepath, path) for path in paths) def parse_defines(apachectl): @@ -144,7 +144,7 @@ def parse_defines(apachectl): :rtype: dict """ - variables = dict() + variables = {} define_cmd = [apachectl, "-t", "-D", "DUMP_RUN_CFG"] matches = parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)") diff --git a/certbot-apache/certbot_apache/_internal/assertions.py b/certbot-apache/certbot_apache/_internal/assertions.py index e1b4cdcc8..2b2ce4f50 100644 --- a/certbot-apache/certbot_apache/_internal/assertions.py +++ b/certbot-apache/certbot_apache/_internal/assertions.py @@ -132,9 +132,9 @@ def assertEqualPathsList(first, second): # pragma: no cover Checks that the two lists of file paths match. This assertion allows for wildcard paths. """ - if any([isPass(path) for path in first]): + if any(isPass(path) for path in first): return - if any([isPass(path) for path in second]): + if any(isPass(path) for path in second): return for fpath in first: assert any([fnmatch.fnmatch(fpath, spath) for spath in second]) diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index f85d80923..3b2ce40d8 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -339,7 +339,7 @@ class AugeasBlockNode(AugeasDirectiveNode): def find_blocks(self, name, exclude=True): """Recursive search of BlockNodes from the sequence of children""" - nodes = list() + nodes = [] paths = self._aug_find_blocks(name) if exclude: paths = self.parser.exclude_dirs(paths) @@ -351,7 +351,7 @@ class AugeasBlockNode(AugeasDirectiveNode): def find_directives(self, name, exclude=True): """Recursive search of DirectiveNodes from the sequence of children""" - nodes = list() + nodes = [] ownpath = self.metadata.get("augeaspath") directives = self.parser.find_dir(name, start=ownpath, exclude=exclude) @@ -374,7 +374,7 @@ class AugeasBlockNode(AugeasDirectiveNode): :param str comment: Comment content to search for. """ - nodes = list() + nodes = [] ownpath = self.metadata.get("augeaspath") comments = self.parser.find_comments(comment, start=ownpath) diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 86d579954..dbfc15468 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -208,12 +208,12 @@ class ApacheConfigurator(common.Installer): super(ApacheConfigurator, self).__init__(*args, **kwargs) # Add name_server association dict - self.assoc = dict() # type: Dict[str, obj.VirtualHost] + self.assoc = {} # type: Dict[str, obj.VirtualHost] # Outstanding challenges self._chall_out = set() # type: Set[KeyAuthorizationAnnotatedChallenge] # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = dict() # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] # Temporary state for AutoHSTS enhancement @@ -436,7 +436,7 @@ class ApacheConfigurator(common.Installer): """Initializes the ParserNode parser root instance.""" if HAS_APACHECONFIG: - apache_vars = dict() + apache_vars = {} apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) @@ -548,7 +548,7 @@ class ApacheConfigurator(common.Installer): # Go through the vhosts, making sure that we cover all the names # present, but preferring the SSL vhosts - filtered_vhosts = dict() + filtered_vhosts = {} for vhost in vhosts: for name in vhost.get_names(): if vhost.ssl: @@ -574,7 +574,7 @@ class ApacheConfigurator(common.Installer): # Make sure we create SSL vhosts for the ones that are HTTP only # if requested. - return_vhosts = list() + return_vhosts = [] for vhost in dialog_output: if not vhost.ssl: return_vhosts.append(self.make_vhost_ssl(vhost)) @@ -1579,7 +1579,7 @@ class ApacheConfigurator(common.Installer): result.append(comment) sift = True - result.append('\n'.join(['# ' + l for l in chunk])) + result.append('\n'.join('# ' + l for l in chunk)) else: result.append('\n'.join(chunk)) return result, sift @@ -1719,7 +1719,7 @@ class ApacheConfigurator(common.Installer): for addr in vhost.addrs: # In Apache 2.2, when a NameVirtualHost directive is not # set, "*" and "_default_" will conflict when sharing a port - addrs = set((addr,)) + addrs = {addr,} if addr.get_addr() in ("*", "_default_"): addrs.update(obj.Addr((a, addr.get_port(),)) for a in ("*", "_default_")) @@ -1910,7 +1910,7 @@ class ApacheConfigurator(common.Installer): try: self._autohsts = self.storage.fetch("autohsts") except KeyError: - self._autohsts = dict() + self._autohsts = {} def _autohsts_save_state(self): """ @@ -2471,7 +2471,7 @@ class ApacheConfigurator(common.Installer): if len(matches) != 1: raise errors.PluginError("Unable to find Apache version") - return tuple([int(i) for i in matches[0].split(".")]) + return tuple(int(i) for i in matches[0].split(".")) def more_info(self): """Human-readable string to help understand the module""" diff --git a/certbot-apache/certbot_apache/_internal/display_ops.py b/certbot-apache/certbot_apache/_internal/display_ops.py index 1ae32bb47..dabf20606 100644 --- a/certbot-apache/certbot_apache/_internal/display_ops.py +++ b/certbot-apache/certbot_apache/_internal/display_ops.py @@ -21,7 +21,7 @@ def select_vhost_multiple(vhosts): :rtype: :class:`list`of type `~obj.Vhost` """ if not vhosts: - return list() + return [] tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] # Remove the extra newline from the last entry if tags_list: @@ -37,7 +37,7 @@ def select_vhost_multiple(vhosts): def _reversemap_vhosts(names, vhosts): """Helper function for select_vhost_multiple for mapping string representations back to actual vhost objects""" - return_vhosts = list() + return_vhosts = [] for selection in names: for vhost in vhosts: diff --git a/certbot-apache/certbot_apache/_internal/dualparser.py b/certbot-apache/certbot_apache/_internal/dualparser.py index aa66cf84c..eef8f2a0e 100644 --- a/certbot-apache/certbot_apache/_internal/dualparser.py +++ b/certbot-apache/certbot_apache/_internal/dualparser.py @@ -49,7 +49,7 @@ class DualNodeBase(object): pass_primary = assertions.isPassNodeList(primary_res) pass_secondary = assertions.isPassNodeList(secondary_res) - new_nodes = list() + new_nodes = [] if pass_primary and pass_secondary: # Both unimplemented @@ -221,7 +221,7 @@ class DualBlockNode(DualNodeBase): implementations to a list of tuples. """ - matched = list() + matched = [] for p in primary_list: match = None for s in secondary_list: diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 82d7df6aa..f6aa3fe48 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -30,7 +30,7 @@ class ApacheParser(object): """ arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") - fnmatch_chars = set(["*", "?", "\\", "[", "]"]) + fnmatch_chars = {"*", "?", "\\", "[", "]"} def __init__(self, root, vhostroot=None, version=(2, 4), configurator=None): @@ -945,8 +945,8 @@ def case_i(string): :param str string: string to make case i regex """ - return "".join(["[" + c.upper() + c.lower() + "]" - if c.isalpha() else c for c in re.escape(string)]) + return "".join("[" + c.upper() + c.lower() + "]" + if c.isalpha() else c for c in re.escape(string)) def get_aug_path(file_path): diff --git a/certbot-apache/certbot_apache/_internal/parsernode_util.py b/certbot-apache/certbot_apache/_internal/parsernode_util.py index d9646862a..0e39ec365 100644 --- a/certbot-apache/certbot_apache/_internal/parsernode_util.py +++ b/certbot-apache/certbot_apache/_internal/parsernode_util.py @@ -11,7 +11,7 @@ def validate_kwargs(kwargs, required_names): :param list required_names: List of required parameter names. """ - validated_kwargs = dict() + validated_kwargs = {} for name in required_names: try: validated_kwargs[name] = kwargs.pop(name) diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 4468a838e..270cc8c44 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -2,7 +2,6 @@ import mock import os -import unittest import util from certbot import errors diff --git a/certbot-apache/tests/centos6_test.py b/certbot-apache/tests/centos6_test.py index 15d086600..27b4f8e80 100644 --- a/certbot-apache/tests/centos6_test.py +++ b/certbot-apache/tests/centos6_test.py @@ -19,12 +19,12 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "test.example.com.conf"), os.path.join(aug_pre, "test.example.com.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "test.example.com"), obj.VirtualHost( os.path.join(prefix, "ssl.conf"), os.path.join(aug_pre, "ssl.conf/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), + {obj.Addr.fromstring("_default_:443")}, True, True, None) ] return vh_truth @@ -104,7 +104,7 @@ class CentOS6Tests(util.ApacheTest): pre_loadmods = self.config.parser.find_dir( "LoadModule", "ssl_module", exclude=False) # LoadModules are not within IfModule blocks - self.assertFalse(any(["ifmodule" in m.lower() for m in pre_loadmods])) + self.assertFalse(any("ifmodule" in m.lower() for m in pre_loadmods)) self.config.assoc["test.example.com"] = self.vh_truth[0] self.config.deploy_cert( "random.demo", "example/cert.pem", "example/key.pem", diff --git a/certbot-apache/tests/centos_test.py b/certbot-apache/tests/centos_test.py index 5e334c51e..40d1361d4 100644 --- a/certbot-apache/tests/centos_test.py +++ b/certbot-apache/tests/centos_test.py @@ -21,12 +21,12 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "centos.example.com.conf"), os.path.join(aug_pre, "centos.example.com.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "centos.example.com"), obj.VirtualHost( os.path.join(prefix, "ssl.conf"), os.path.join(aug_pre, "ssl.conf/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), + {obj.Addr.fromstring("_default_:443")}, True, True, None) ] return vh_truth diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 54f4c9251..694c1c0bb 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -99,7 +99,7 @@ class MultipleVhostsTest(util.ApacheTest): parserargs = ["server_root", "enmod", "dismod", "le_vhost_ext", "vhost_root", "logs_root", "challenge_location", "handle_modules", "handle_sites", "ctl"] - exp = dict() + exp = {} for k in ApacheConfigurator.OS_DEFAULTS: if k in parserargs: @@ -140,11 +140,9 @@ class MultipleVhostsTest(util.ApacheTest): mock_utility = mock_getutility() mock_utility.notification = mock.MagicMock(return_value=True) names = self.config.get_all_names() - self.assertEqual(names, set( - ["certbot.demo", "ocspvhost.com", "encryption-example.demo", + self.assertEqual(names, {"certbot.demo", "ocspvhost.com", "encryption-example.demo", "nonsym.link", "vhost.in.rootconf", "www.certbot.demo", - "duplicate.example.com"] - )) + "duplicate.example.com"}) @certbot_util.patch_get_utility() @mock.patch("certbot_apache._internal.configurator.socket.gethostbyaddr") @@ -154,9 +152,9 @@ class MultipleVhostsTest(util.ApacheTest): mock_utility.notification.return_value = True vhost = obj.VirtualHost( "fp", "ap", - set([obj.Addr(("8.8.8.8", "443")), + {obj.Addr(("8.8.8.8", "443")), obj.Addr(("zombo.com",)), - obj.Addr(("192.168.1.2"))]), + obj.Addr(("192.168.1.2"))}, True, False) self.config.vhosts.append(vhost) @@ -185,7 +183,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_bad_servername_alias(self): ssl_vh1 = obj.VirtualHost( - "fp1", "ap1", set([obj.Addr(("*", "443"))]), + "fp1", "ap1", {obj.Addr(("*", "443"))}, True, False) # pylint: disable=protected-access self.config._add_servernames(ssl_vh1) @@ -198,7 +196,7 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.config._add_servernames(self.vh_truth[2]) self.assertEqual( - self.vh_truth[2].get_names(), set(["*.le.co", "ip-172-30-0-17"])) + self.vh_truth[2].get_names(), {"*.le.co", "ip-172-30-0-17"}) def test_get_virtual_hosts(self): """Make sure all vhosts are being properly found.""" @@ -269,7 +267,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_choose_vhost_select_vhost_conflicting_non_ssl(self, mock_select): mock_select.return_value = self.vh_truth[3] conflicting_vhost = obj.VirtualHost( - "path", "aug_path", set([obj.Addr.fromstring("*:443")]), + "path", "aug_path", {obj.Addr.fromstring("*:443")}, True, True) self.config.vhosts.append(conflicting_vhost) @@ -278,14 +276,14 @@ class MultipleVhostsTest(util.ApacheTest): def test_find_best_http_vhost_default(self): vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr.fromstring("_default_:80")]), False, True) + "fp", "ap", {obj.Addr.fromstring("_default_:80")}, False, True) self.config.vhosts = [vh] self.assertEqual(self.config.find_best_http_vhost("foo.bar", False), vh) def test_find_best_http_vhost_port(self): port = "8080" vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr.fromstring("*:" + port)]), + "fp", "ap", {obj.Addr.fromstring("*:" + port)}, False, True, "encryption-example.demo") self.config.vhosts.append(vh) self.assertEqual(self.config.find_best_http_vhost("foo.bar", False, port), vh) @@ -313,8 +311,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_find_best_vhost_variety(self): # pylint: disable=protected-access ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), - obj.Addr(("zombo.com",))]), + "fp", "ap", {obj.Addr(("*", "443")), + obj.Addr(("zombo.com",))}, True, False) self.config.vhosts.append(ssl_vh) self.assertEqual(self.config._find_best_vhost("zombo.com"), ssl_vh) @@ -685,7 +683,7 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(ssl_vhost.path, "/files" + ssl_vhost.filep + "/IfModule/Virtualhost") self.assertEqual(len(ssl_vhost.addrs), 1) - self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs) + self.assertEqual({obj.Addr.fromstring("*:443")}, ssl_vhost.addrs) self.assertEqual(ssl_vhost.name, "encryption-example.demo") self.assertTrue(ssl_vhost.ssl) self.assertFalse(ssl_vhost.enabled) @@ -910,7 +908,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.parser.modules["rewrite_module"] = None mock_exe.return_value = True ssl_vh1 = obj.VirtualHost( - "fp1", "ap1", set([obj.Addr(("*", "443"))]), + "fp1", "ap1", {obj.Addr(("*", "443"))}, True, False) ssl_vh1.name = "satoshi.com" self.config.vhosts.append(ssl_vh1) @@ -1011,7 +1009,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_get_http_vhost_third_filter(self): ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443"))]), + "fp", "ap", {obj.Addr(("*", "443"))}, True, False) ssl_vh.name = "satoshi.com" self.config.vhosts.append(ssl_vh) @@ -1214,8 +1212,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_with_conflict(self): self.config.parser.modules["rewrite_module"] = None ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), - obj.Addr(("zombo.com",))]), + "fp", "ap", {obj.Addr(("*", "443")), + obj.Addr(("zombo.com",))}, True, False) # No names ^ this guy should conflict. @@ -1257,7 +1255,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.get_version = mock.Mock(return_value=(2, 3, 9)) # For full testing... give names... self.vh_truth[1].name = "default.com" - self.vh_truth[1].aliases = set(["yes.default.com"]) + self.vh_truth[1].aliases = {"yes.default.com"} # pylint: disable=protected-access self.config._enable_redirect(self.vh_truth[1], "") @@ -1268,7 +1266,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.get_version = mock.Mock(return_value=(2, 2)) # For full testing... give names... self.vh_truth[1].name = "default.com" - self.vh_truth[1].aliases = set(["yes.default.com"]) + self.vh_truth[1].aliases = {"yes.default.com"} # pylint: disable=protected-access self.config._enable_redirect(self.vh_truth[1], "") @@ -1609,7 +1607,7 @@ class MultiVhostsTest(util.ApacheTest): self.assertEqual(ssl_vhost.path, "/files" + ssl_vhost.filep + "/IfModule/VirtualHost") self.assertEqual(len(ssl_vhost.addrs), 1) - self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs) + self.assertEqual({obj.Addr.fromstring("*:443")}, ssl_vhost.addrs) self.assertEqual(ssl_vhost.name, "banana.vomit.com") self.assertTrue(ssl_vhost.ssl) self.assertFalse(ssl_vhost.enabled) diff --git a/certbot-apache/tests/display_ops_test.py b/certbot-apache/tests/display_ops_test.py index 50bdc03cf..e9f54a562 100644 --- a/certbot-apache/tests/display_ops_test.py +++ b/certbot-apache/tests/display_ops_test.py @@ -93,9 +93,9 @@ class SelectVhostTest(unittest.TestCase): self.vhosts.append( obj.VirtualHost( - "path", "aug_path", set([obj.Addr.fromstring("*:80")]), + "path", "aug_path", {obj.Addr.fromstring("*:80")}, False, False, - "wildcard.com", set(["*.wildcard.com"]))) + "wildcard.com", {"*.wildcard.com"})) self.assertEqual(self.vhosts[5], self._call(self.vhosts)) diff --git a/certbot-apache/tests/gentoo_test.py b/certbot-apache/tests/gentoo_test.py index 3b35dde54..5a35c17ae 100644 --- a/certbot-apache/tests/gentoo_test.py +++ b/certbot-apache/tests/gentoo_test.py @@ -21,19 +21,19 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "gentoo.example.com.conf"), os.path.join(aug_pre, "gentoo.example.com.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "gentoo.example.com"), obj.VirtualHost( os.path.join(prefix, "00_default_vhost.conf"), os.path.join(aug_pre, "00_default_vhost.conf/IfDefine/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "localhost"), obj.VirtualHost( os.path.join(prefix, "00_default_ssl_vhost.conf"), os.path.join(aug_pre, "00_default_ssl_vhost.conf" + "/IfDefine/IfDefine/IfModule/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), + {obj.Addr.fromstring("_default_:443")}, True, True, "localhost") ] return vh_truth diff --git a/certbot-apache/tests/obj_test.py b/certbot-apache/tests/obj_test.py index 1761b9c94..eac6a64ef 100644 --- a/certbot-apache/tests/obj_test.py +++ b/certbot-apache/tests/obj_test.py @@ -14,13 +14,13 @@ class VirtualHostTest(unittest.TestCase): self.addr_default = Addr.fromstring("_default_:443") self.vhost1 = VirtualHost( - "filep", "vh_path", set([self.addr1]), False, False, "localhost") + "filep", "vh_path", {self.addr1}, False, False, "localhost") self.vhost1b = VirtualHost( - "filep", "vh_path", set([self.addr1]), False, False, "localhost") + "filep", "vh_path", {self.addr1}, False, False, "localhost") self.vhost2 = VirtualHost( - "fp", "vhp", set([self.addr2]), False, False, "localhost") + "fp", "vhp", {self.addr2}, False, False, "localhost") def test_repr(self): self.assertEqual(repr(self.addr2), @@ -42,7 +42,7 @@ class VirtualHostTest(unittest.TestCase): complex_vh = VirtualHost( "fp", "vhp", - set([Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")]), + {Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")}, False, False) self.assertTrue(complex_vh.conflicts([self.addr1])) self.assertTrue(complex_vh.conflicts([self.addr2])) @@ -57,14 +57,14 @@ class VirtualHostTest(unittest.TestCase): def test_same_server(self): from certbot_apache._internal.obj import VirtualHost no_name1 = VirtualHost( - "fp", "vhp", set([self.addr1]), False, False, None) + "fp", "vhp", {self.addr1}, False, False, None) no_name2 = VirtualHost( - "fp", "vhp", set([self.addr2]), False, False, None) + "fp", "vhp", {self.addr2}, False, False, None) no_name3 = VirtualHost( - "fp", "vhp", set([self.addr_default]), + "fp", "vhp", {self.addr_default}, False, False, None) no_name4 = VirtualHost( - "fp", "vhp", set([self.addr2, self.addr_default]), + "fp", "vhp", {self.addr2, self.addr_default}, False, False, None) self.assertTrue(self.vhost1.same_server(self.vhost2)) diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index 4994103f3..a39e5225d 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -142,71 +142,71 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "encryption-example.conf"), os.path.join(aug_pre, "encryption-example.conf/Virtualhost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "encryption-example.demo"), obj.VirtualHost( os.path.join(prefix, "default-ssl.conf"), os.path.join(aug_pre, "default-ssl.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), True, True), + {obj.Addr.fromstring("_default_:443")}, True, True), obj.VirtualHost( os.path.join(prefix, "000-default.conf"), os.path.join(aug_pre, "000-default.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80"), - obj.Addr.fromstring("[::]:80")]), + {obj.Addr.fromstring("*:80"), + obj.Addr.fromstring("[::]:80")}, False, True, "ip-172-30-0-17"), obj.VirtualHost( os.path.join(prefix, "certbot.conf"), os.path.join(aug_pre, "certbot.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "certbot.demo", aliases=["www.certbot.demo"]), obj.VirtualHost( os.path.join(prefix, "mod_macro-example.conf"), os.path.join(aug_pre, "mod_macro-example.conf/Macro/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, modmacro=True), obj.VirtualHost( os.path.join(prefix, "default-ssl-port-only.conf"), os.path.join(aug_pre, ("default-ssl-port-only.conf/" "IfModule/VirtualHost")), - set([obj.Addr.fromstring("_default_:443")]), True, True), + {obj.Addr.fromstring("_default_:443")}, True, True), obj.VirtualHost( os.path.join(prefix, "wildcard.conf"), os.path.join(aug_pre, "wildcard.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "ip-172-30-0-17", aliases=["*.blue.purple.com"]), obj.VirtualHost( os.path.join(prefix, "ocsp-ssl.conf"), os.path.join(aug_pre, "ocsp-ssl.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("10.2.3.4:443")]), True, True, + {obj.Addr.fromstring("10.2.3.4:443")}, True, True, "ocspvhost.com"), obj.VirtualHost( os.path.join(prefix, "non-symlink.conf"), os.path.join(aug_pre, "non-symlink.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "nonsym.link"), obj.VirtualHost( os.path.join(prefix, "default-ssl-port-only.conf"), os.path.join(aug_pre, "default-ssl-port-only.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), True, True, ""), + {obj.Addr.fromstring("*:80")}, True, True, ""), obj.VirtualHost( os.path.join(temp_dir, config_name, "apache2/apache2.conf"), "/files" + os.path.join(temp_dir, config_name, "apache2/apache2.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "vhost.in.rootconf"), obj.VirtualHost( os.path.join(prefix, "duplicatehttp.conf"), os.path.join(aug_pre, "duplicatehttp.conf/VirtualHost"), - set([obj.Addr.fromstring("10.2.3.4:80")]), False, True, + {obj.Addr.fromstring("10.2.3.4:80")}, False, True, "duplicate.example.com"), obj.VirtualHost( os.path.join(prefix, "duplicatehttps.conf"), os.path.join(aug_pre, "duplicatehttps.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("10.2.3.4:443")]), True, True, + {obj.Addr.fromstring("10.2.3.4:443")}, True, True, "duplicate.example.com")] return vh_truth if config_name == "debian_apache_2_4/multi_vhosts": @@ -217,27 +217,27 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "default.conf"), os.path.join(aug_pre, "default.conf/VirtualHost[1]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "ip-172-30-0-17"), obj.VirtualHost( os.path.join(prefix, "default.conf"), os.path.join(aug_pre, "default.conf/VirtualHost[2]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "banana.vomit.com"), obj.VirtualHost( os.path.join(prefix, "multi-vhost.conf"), os.path.join(aug_pre, "multi-vhost.conf/VirtualHost[1]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "1.multi.vhost.tld"), obj.VirtualHost( os.path.join(prefix, "multi-vhost.conf"), os.path.join(aug_pre, "multi-vhost.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "2.multi.vhost.tld"), obj.VirtualHost( os.path.join(prefix, "multi-vhost.conf"), os.path.join(aug_pre, "multi-vhost.conf/VirtualHost[2]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "3.multi.vhost.tld")] return vh_truth return None # pragma: no cover diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index 6f8670000..e295aefd7 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -1,5 +1,4 @@ """Module to handle the context of integration tests.""" -import logging import os import shutil import sys diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index b08f11e89..fbb965034 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -236,7 +236,7 @@ def generate_csr(domains, key_path, csr_path, key_type=RSA_KEY_TYPE): file_h.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, key)) req = crypto.X509Req() - san = ', '.join(['DNS:{0}'.format(item) for item in domains]) + san = ', '.join('DNS:{0}'.format(item) for item in domains) san_constraint = crypto.X509Extension(b'subjectAltName', False, san.encode('utf-8')) req.add_extensions([san_constraint]) diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index d719f583b..5140dc8ea 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -96,7 +96,7 @@ def test_authenticator(plugin, config, temp_dir): def _create_achalls(plugin): """Returns a list of annotated challenges to test on plugin""" - achalls = list() + achalls = [] names = plugin.get_testable_domain_names() for domain in names: prefs = plugin.get_chall_pref(domain) diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index ddab48512..a903c12bf 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -748,7 +748,7 @@ class NginxConfigurator(common.Installer): # if there is no separate SSL block, break the block into two and # choose the SSL block. - if vhost.ssl and any([not addr.ssl for addr in vhost.addrs]): + if vhost.ssl and any(not addr.ssl for addr in vhost.addrs): _, vhost = self._split_block(vhost) header_directives = [ @@ -983,7 +983,7 @@ class NginxConfigurator(common.Installer): logger.warning("NGINX derivative %s is not officially supported by" " certbot", product_name) - nginx_version = tuple([int(i) for i in product_version.split(".")]) + nginx_version = tuple(int(i) for i in product_version.split(".")) # nginx < 0.8.48 uses machine hostname as default server_name instead of # the empty string diff --git a/certbot-nginx/certbot_nginx/_internal/display_ops.py b/certbot-nginx/certbot_nginx/_internal/display_ops.py index bbb47f98a..356cc506c 100644 --- a/certbot-nginx/certbot_nginx/_internal/display_ops.py +++ b/certbot-nginx/certbot_nginx/_internal/display_ops.py @@ -17,7 +17,7 @@ def select_vhost_multiple(vhosts): :rtype: :class:`list`of type `~obj.Vhost` """ if not vhosts: - return list() + return [] tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] # Remove the extra newline from the last entry if tags_list: @@ -33,7 +33,7 @@ def select_vhost_multiple(vhosts): def _reversemap_vhosts(names, vhosts): """Helper function for select_vhost_multiple for mapping string representations back to actual vhost objects""" - return_vhosts = list() + return_vhosts = [] for selection in names: for vhost in vhosts: diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 0c1151826..bb0bb7d6f 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -404,9 +404,9 @@ class NginxParser(object): if directive and directive[0] == 'listen': # Exclude one-time use parameters which will cause an error if repeated. # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen - exclude = set(('default_server', 'default', 'setfib', 'fastopen', 'backlog', + exclude = {'default_server', 'default', 'setfib', 'fastopen', 'backlog', 'rcvbuf', 'sndbuf', 'accept_filter', 'deferred', 'bind', - 'ipv6only', 'reuseport', 'so_keepalive')) + 'ipv6only', 'reuseport', 'so_keepalive'} for param in exclude: # See: github.com/certbot/certbot/pull/6223#pullrequestreview-143019225 @@ -578,7 +578,7 @@ def _update_or_add_directives(directives, insert_at_top, block): INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite', 'add_header']) +REPEATABLE_DIRECTIVES = {'server_name', 'listen', INCLUDE, 'rewrite', 'add_header'} COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 61b31b2d5..390e18e4d 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -206,7 +206,7 @@ class Sentence(Parsable): :returns: whether this lists is parseable by `Sentence`. """ return isinstance(lists, list) and len(lists) > 0 and \ - all([isinstance(elem, six.string_types) for elem in lists]) + all(isinstance(elem, six.string_types) for elem in lists) def parse(self, raw_list, add_spaces=False): """ Parses a list of string types into this object. @@ -214,7 +214,7 @@ class Sentence(Parsable): if add_spaces: raw_list = _space_list(raw_list) if not isinstance(raw_list, list) or \ - any([not isinstance(elem, six.string_types) for elem in raw_list]): + any(not isinstance(elem, six.string_types) for elem in raw_list): raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") self._data = raw_list diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 0d9d6d356..0a04a22a4 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -104,7 +104,7 @@ class NginxConfiguratorTest(util.NginxTest): filep = self.config.parser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(filep, None, None, None, - set(['.example.com', 'example.*']), + {'.example.com', 'example.*'}, None, [0]) self.config.parser.add_server_directives( mock_vhost, @@ -150,11 +150,11 @@ class NginxConfiguratorTest(util.NginxTest): self._test_choose_vhosts_common('ipv6.com', 'ipv6_conf') def _test_choose_vhosts_common(self, name, conf): - conf_names = {'localhost_conf': set(['localhost', r'~^(www\.)?(example|bar)\.']), - 'server_conf': set(['somename', 'another.alias', 'alias']), - 'example_conf': set(['.example.com', 'example.*']), - 'foo_conf': set(['*.www.foo.com', '*.www.example.com']), - 'ipv6_conf': set(['ipv6.com'])} + conf_names = {'localhost_conf': {'localhost', r'~^(www\.)?(example|bar)\.'}, + 'server_conf': {'somename', 'another.alias', 'alias'}, + 'example_conf': {'.example.com', 'example.*'}, + 'foo_conf': {'*.www.foo.com', '*.www.example.com'}, + 'ipv6_conf': {'ipv6.com'}} conf_path = {'localhost': "etc_nginx/nginx.conf", 'alias': "etc_nginx/nginx.conf", @@ -177,7 +177,7 @@ class NginxConfiguratorTest(util.NginxTest): self.assertTrue(vhost.ipv6_enabled()) # Make sure that we have SSL enabled also for IPv6 addr self.assertTrue( - any([True for x in vhost.addrs if x.ssl and x.ipv6])) + any(True for x in vhost.addrs if x.ssl and x.ipv6)) def test_choose_vhosts_bad(self): bad_results = ['www.foo.com', 'example', 't.www.bar.co', diff --git a/certbot-nginx/tests/obj_test.py b/certbot-nginx/tests/obj_test.py index db808229f..93b9493eb 100644 --- a/certbot-nginx/tests/obj_test.py +++ b/certbot-nginx/tests/obj_test.py @@ -98,10 +98,10 @@ class AddrTest(unittest.TestCase): def test_set_inclusion(self): from certbot_nginx._internal.obj import Addr - set_a = set([self.addr1, self.addr2]) + set_a = {self.addr1, self.addr2} addr1b = Addr.fromstring("192.168.1.1") addr2b = Addr.fromstring("192.168.1.1:* ssl") - set_b = set([addr1b, addr2b]) + set_b = {addr1b, addr2b} self.assertEqual(set_a, set_b) @@ -120,8 +120,8 @@ class VirtualHostTest(unittest.TestCase): ] self.vhost1 = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw1, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw1, []) raw2 = [ ['listen', '69.50.225.155:9000'], [['if', '($scheme', '!=', '"https") '], @@ -130,24 +130,24 @@ class VirtualHostTest(unittest.TestCase): ] self.vhost2 = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw2, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw2, []) raw3 = [ ['listen', '69.50.225.155:9000'], ['rewrite', '^(.*)$', '$scheme://www.domain.com$1', 'permanent'] ] self.vhost3 = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw3, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw3, []) raw4 = [ ['listen', '69.50.225.155:9000'], ['server_name', 'return.com'] ] self.vhost4 = VirtualHost( "filp", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw4, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw4, []) raw_has_hsts = [ ['listen', '69.50.225.155:9000'], ['server_name', 'return.com'], @@ -155,16 +155,16 @@ class VirtualHostTest(unittest.TestCase): ] self.vhost_has_hsts = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw_has_hsts, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw_has_hsts, []) def test_eq(self): from certbot_nginx._internal.obj import Addr from certbot_nginx._internal.obj import VirtualHost vhost1b = VirtualHost( "filep", - set([Addr.fromstring("localhost blah")]), False, False, - set(['localhost']), [], []) + {Addr.fromstring("localhost blah")}, False, False, + {'localhost'}, [], []) self.assertEqual(vhost1b, self.vhost1) self.assertEqual(str(vhost1b), str(self.vhost1)) @@ -203,8 +203,8 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], []] vhost_haystack = VirtualHost( "filp", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), test_haystack, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, test_haystack, []) test_bad_haystack = [['listen', '80'], ['root', '/var/www/html'], ['index', 'index.html index.htm index.nginx-debian.html'], ['server_name', 'two.functorkitten.xyz'], ['listen', '443 ssl'], @@ -219,8 +219,8 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], []] vhost_bad_haystack = VirtualHost( "filp", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), test_bad_haystack, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, test_bad_haystack, []) self.assertTrue(vhost_haystack.contains_list(test_needle)) self.assertFalse(vhost_bad_haystack.contains_list(test_needle)) diff --git a/certbot-nginx/tests/parser_obj_test.py b/certbot-nginx/tests/parser_obj_test.py index 132f83771..bb7834701 100644 --- a/certbot-nginx/tests/parser_obj_test.py +++ b/certbot-nginx/tests/parser_obj_test.py @@ -80,7 +80,7 @@ class ParsingHooksTest(unittest.TestCase): fake_parser1.should_parse = lambda x: False parsing_hooks.return_value = (fake_parser1,) self.assertRaises(errors.MisconfigurationError, parse_raw, []) - parsing_hooks.return_value = tuple() + parsing_hooks.return_value = () self.assertRaises(errors.MisconfigurationError, parse_raw, []) def test_parse_raw_passes_add_spaces(self): diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 72cfc0716..21dd1043d 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -126,7 +126,7 @@ class NginxParserTest(util.NginxTest): vhost = obj.VirtualHost(nparser.abs_path('sites-enabled/globalssl.com'), [obj.Addr('4.8.2.6', '57', True, False, False, False)], - True, True, set(['globalssl.com']), [], [0]) + True, True, {'globalssl.com'}, [], [0]) globalssl_com = [x for x in vhosts if 'globalssl.com' in x.filep][0] self.assertEqual(vhost, globalssl_com) @@ -139,8 +139,8 @@ class NginxParserTest(util.NginxTest): [obj.Addr('', '8080', False, False, False, False)], False, True, - set(['localhost', - r'~^(www\.)?(example|bar)\.']), + {'localhost', + r'~^(www\.)?(example|bar)\.'}, [], [10, 1, 9]) vhost2 = obj.VirtualHost(nparser.abs_path('nginx.conf'), [obj.Addr('somename', '8080', False, False, @@ -148,7 +148,7 @@ class NginxParserTest(util.NginxTest): obj.Addr('', '8000', False, False, False, False)], False, True, - set(['somename', 'another.alias', 'alias']), + {'somename', 'another.alias', 'alias'}, [], [10, 1, 12]) vhost3 = obj.VirtualHost(nparser.abs_path('sites-enabled/example.com'), [obj.Addr('69.50.225.155', '9000', @@ -156,19 +156,19 @@ class NginxParserTest(util.NginxTest): obj.Addr('127.0.0.1', '', False, False, False, False)], False, True, - set(['.example.com', 'example.*']), [], [0]) + {'.example.com', 'example.*'}, [], [0]) vhost4 = obj.VirtualHost(nparser.abs_path('sites-enabled/default'), [obj.Addr('myhost', '', False, True, False, False), obj.Addr('otherhost', '', False, True, False, False)], - False, True, set(['www.example.org']), + False, True, {'www.example.org'}, [], [0]) vhost5 = obj.VirtualHost(nparser.abs_path('foo.conf'), [obj.Addr('*', '80', True, True, False, False)], - True, True, set(['*.www.foo.com', - '*.www.example.com']), + True, True, {'*.www.foo.com', + '*.www.example.com'}, [], [2, 1, 0]) self.assertEqual(14, len(vhosts)) @@ -208,11 +208,11 @@ class NginxParserTest(util.NginxTest): nparser = parser.NginxParser(self.config_path) mock_vhost = obj.VirtualHost(nparser.abs_path('nginx.conf'), None, None, None, - set(['localhost', - r'~^(www\.)?(example|bar)\.']), + {'localhost', + r'~^(www\.)?(example|bar)\.'}, None, [10, 1, 9]) example_com = nparser.abs_path('sites-enabled/example.com') - names = set(['.example.com', 'example.*']) + names = {'.example.com', 'example.*'} mock_vhost.filep = example_com mock_vhost.names = names mock_vhost.path = [0] @@ -232,8 +232,8 @@ class NginxParserTest(util.NginxTest): nparser = parser.NginxParser(self.config_path) mock_vhost = obj.VirtualHost(nparser.abs_path('nginx.conf'), None, None, None, - set(['localhost', - r'~^(www\.)?(example|bar)\.']), + {'localhost', + r'~^(www\.)?(example|bar)\.'}, None, [10, 1, 9]) nparser.add_server_directives(mock_vhost, [['foo', 'bar'], ['\n ', 'ssl_certificate', ' ', @@ -243,7 +243,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual(1, len(re.findall(ssl_re, dump))) example_com = nparser.abs_path('sites-enabled/example.com') - names = set(['.example.com', 'example.*']) + names = {'.example.com', 'example.*'} mock_vhost.filep = example_com mock_vhost.names = names mock_vhost.path = [0] @@ -264,7 +264,7 @@ class NginxParserTest(util.NginxTest): ]]]) server_conf = nparser.abs_path('server.conf') - names = set(['alias', 'another.alias', 'somename']) + names = {'alias', 'another.alias', 'somename'} mock_vhost.filep = server_conf mock_vhost.names = names mock_vhost.path = [] @@ -279,7 +279,7 @@ class NginxParserTest(util.NginxTest): example_com = nparser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(example_com, None, None, None, - set(['.example.com', 'example.*']), + {'.example.com', 'example.*'}, None, [0]) nparser.add_server_directives(mock_vhost, [['\n ', '#', ' ', 'what a nice comment']]) @@ -301,7 +301,7 @@ class NginxParserTest(util.NginxTest): def test_replace_server_directives(self): nparser = parser.NginxParser(self.config_path) - target = set(['.example.com', 'example.*']) + target = {'.example.com', 'example.*'} filep = nparser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(filep, None, None, None, target, None, [0]) nparser.update_or_add_server_directives( @@ -314,7 +314,7 @@ class NginxParserTest(util.NginxTest): ['server_name', 'foobar.com'], ['#', COMMENT], ['server_name', 'example.*'], [] ]]]) - mock_vhost.names = set(['foobar.com', 'example.*']) + mock_vhost.names = {'foobar.com', 'example.*'} nparser.update_or_add_server_directives( mock_vhost, [['ssl_certificate', 'cert.pem']]) self.assertEqual( @@ -328,19 +328,19 @@ class NginxParserTest(util.NginxTest): def test_get_best_match(self): target_name = 'www.eff.org' - names = [set(['www.eff.org', 'irrelevant.long.name.eff.org', '*.org']), - set(['eff.org', 'ww2.eff.org', 'test.www.eff.org']), - set(['*.eff.org', '.www.eff.org']), - set(['.eff.org', '*.org']), - set(['www.eff.', 'www.eff.*', '*.www.eff.org']), - set(['example.com', r'~^(www\.)?(eff.+)', '*.eff.*']), - set(['*', r'~^(www\.)?(eff.+)']), - set(['www.*', r'~^(www\.)?(eff.+)', '.test.eff.org']), - set(['*.org', r'*.eff.org', 'www.eff.*']), - set(['*.www.eff.org', 'www.*']), - set(['*.org']), - set([]), - set(['example.com'])] + names = [{'www.eff.org', 'irrelevant.long.name.eff.org', '*.org'}, + {'eff.org', 'ww2.eff.org', 'test.www.eff.org'}, + {'*.eff.org', '.www.eff.org'}, + {'.eff.org', '*.org'}, + {'www.eff.', 'www.eff.*', '*.www.eff.org'}, + {'example.com', r'~^(www\.)?(eff.+)', '*.eff.*'}, + {'*', r'~^(www\.)?(eff.+)'}, + {'www.*', r'~^(www\.)?(eff.+)', '.test.eff.org'}, + {'*.org', r'*.eff.org', 'www.eff.*'}, + {'*.www.eff.org', 'www.*'}, + {'*.org'}, + set(), + {'example.com'}] winners = [('exact', 'www.eff.org'), (None, None), ('exact', '.www.eff.org'), diff --git a/certbot/certbot/_internal/cli/cli_constants.py b/certbot/certbot/_internal/cli/cli_constants.py index 748ae0d94..4bc84bfe7 100644 --- a/certbot/certbot/_internal/cli/cli_constants.py +++ b/certbot/certbot/_internal/cli/cli_constants.py @@ -91,17 +91,17 @@ ARGPARSE_PARAMS_TO_REMOVE = ("const", "nargs", "type",) # These sets are used when to help detect options set by the user. -EXIT_ACTIONS = set(("help", "version",)) +EXIT_ACTIONS = {"help", "version",} -ZERO_ARG_ACTIONS = set(("store_const", "store_true", - "store_false", "append_const", "count",)) +ZERO_ARG_ACTIONS = {"store_const", "store_true", + "store_false", "append_const", "count",} # Maps a config option to a set of config options that may have modified it. # This dictionary is used recursively, so if A modifies B and B modifies C, # it is determined that C was modified by the user if A was modified. -VAR_MODIFIERS = {"account": set(("server",)), - "renew_hook": set(("deploy_hook",)), - "server": set(("dry_run", "staging",)), - "webroot_map": set(("webroot_path",))} +VAR_MODIFIERS = {"account": {"server",}, + "renew_hook": {"deploy_hook",}, + "server": {"dry_run", "staging",}, + "webroot_map": {"webroot_path",}} diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 94ccb8b5e..264f9667e 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -890,7 +890,7 @@ def enhance(config, plugins): """ supported_enhancements = ["hsts", "redirect", "uir", "staple"] # Check that at least one enhancement was requested on command line - oldstyle_enh = any([getattr(config, enh) for enh in supported_enhancements]) + oldstyle_enh = any(getattr(config, enh) for enh in supported_enhancements) if not enhancements.are_requested(config) and not oldstyle_enh: msg = ("Please specify one or more enhancement types to configure. To list " "the available enhancement types, run:\n\n%s --help enhance\n") diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index 87ccdbd7e..b46622796 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -71,7 +71,7 @@ permitted by DNS standards.) super(Authenticator, self).__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() - self.env = dict() \ + self.env = {} \ # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] self.subsequent_dns_challenge = False self.subsequent_any_challenge = False diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index 6d87e4b07..53cef3969 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -138,7 +138,7 @@ def choose_plugin(prepared, question): while True: disp = z_util(interfaces.IDisplay) - if "CERTBOT_AUTO" in os.environ and names == set(("apache", "nginx")): + if "CERTBOT_AUTO" in os.environ and names == {"apache", "nginx"}: # The possibility of being offered exactly apache and nginx here # is new interactivity brought by https://github.com/certbot/certbot/issues/4079, # so set apache as a default for those kinds of non-interactive use diff --git a/certbot/certbot/display/ops.py b/certbot/certbot/display/ops.py index f24f6ed99..21d169a55 100644 --- a/certbot/certbot/display/ops.py +++ b/certbot/certbot/display/ops.py @@ -195,7 +195,7 @@ def _choose_names_manually(prompt_prefix=""): cli_flag="--domains", force_interactive=True) if code == display_util.OK: - invalid_domains = dict() + invalid_domains = {} retry_message = "" try: domain_list = display_util.separate_list_input(input_) diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index fe3a97dd5..9123087e7 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -42,7 +42,7 @@ class PluginStorage(object): :raises .errors.PluginStorageError: when unable to open or read the file """ - data = dict() # type: Dict[str, Any] + data = {} # type: Dict[str, Any] filedata = "" try: with open(self._storagepath, 'r') as fh: @@ -107,7 +107,7 @@ class PluginStorage(object): self._initialize_storage() if not self._classkey in self._data.keys(): - self._data[self._classkey] = dict() + self._data[self._classkey] = {} self._data[self._classkey][key] = value def fetch(self, key): diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 5df7bfcf8..327f1bcbe 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -271,7 +271,7 @@ class ChooseNamesTest(unittest.TestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_valid_return(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = (display_util.OK, ["example.com"]) names = self._call(self.mock_install) @@ -280,7 +280,7 @@ class ChooseNamesTest(unittest.TestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_namees_override_question(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = (display_util.OK, ["example.com"]) names = self._call(self.mock_install, "Custom") self.assertEqual(names, ["example.com"]) @@ -289,14 +289,14 @@ class ChooseNamesTest(unittest.TestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_nothing_selected(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = (display_util.OK, []) self.assertEqual(self._call(self.mock_install), []) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_cancel(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = ( display_util.CANCEL, ["example.com"]) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 615f33406..eccfee7db 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -173,14 +173,14 @@ class FileOutputDisplayTest(unittest.TestCase): code, tag_list = self.displayer.checklist( "msg", TAGS, force_interactive=True) self.assertEqual( - (code, set(tag_list)), (display_util.OK, set(["tag1", "tag2"]))) + (code, set(tag_list)), (display_util.OK, {"tag1", "tag2"})) @mock.patch("certbot.display.util.input_with_timeout") def test_checklist_empty(self, mock_input): mock_input.return_value = "" code, tag_list = self.displayer.checklist("msg", TAGS, force_interactive=True) self.assertEqual( - (code, set(tag_list)), (display_util.OK, set(["tag1", "tag2", "tag3"]))) + (code, set(tag_list)), (display_util.OK, {"tag1", "tag2", "tag3"})) @mock.patch("certbot.display.util.input_with_timeout") def test_checklist_miss_valid(self, mock_input): @@ -212,9 +212,9 @@ class FileOutputDisplayTest(unittest.TestCase): ["2", "3"], ] exp = [ - set(["tag1"]), - set(["tag1", "tag2"]), - set(["tag2", "tag3"]), + {"tag1"}, + {"tag1", "tag2"}, + {"tag2", "tag3"}, ] for i, list_ in enumerate(indices): set_tags = set( diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 011313208..899dbc611 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -42,7 +42,7 @@ class ErrorHandlerTest(unittest.TestCase): from certbot._internal import error_handler self.init_func = mock.MagicMock() - self.init_args = set((42,)) + self.init_args = {42,} self.init_kwargs = {'foo': 'bar'} self.handler = error_handler.ErrorHandler(self.init_func, *self.init_args, diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 54d0fcf67..3b7a94489 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -24,7 +24,7 @@ class ValidateHooksTest(unittest.TestCase): self._call(config) types = [call[0][1] for call in mock_validate_hook.call_args_list] - self.assertEqual(set(("pre", "post", "deploy",)), set(types[:-1])) + self.assertEqual({"pre", "post", "deploy",}, set(types[:-1])) # This ensures error messages are about deploy hooks when appropriate self.assertEqual("renew", types[-1]) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 8b2645876..92afc3fef 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1583,9 +1583,9 @@ class EnhanceTest(test_util.ConfigTestCase): not_req_enh = ["uir"] self.assertTrue(mock_client.enhance_config.called) self.assertTrue( - all([getattr(mock_client.config, e) for e in req_enh])) + all(getattr(mock_client.config, e) for e in req_enh)) self.assertFalse( - any([getattr(mock_client.config, e) for e in not_req_enh])) + any(getattr(mock_client.config, e) for e in not_req_enh)) self.assertTrue( "example.com" in mock_client.enhance_config.call_args[0][0]) diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 7543f28f3..1f339a6f3 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -94,7 +94,7 @@ class InstallerTest(test_util.ConfigTestCase): self.reverter = self.installer.reverter def test_add_to_real_checkpoint(self): - files = set(("foo.bar", "baz.qux",)) + files = {"foo.bar", "baz.qux",} save_notes = "foo bar baz qux" self._test_wrapped_method("add_to_checkpoint", files, save_notes) @@ -105,7 +105,7 @@ class InstallerTest(test_util.ConfigTestCase): self._test_add_to_checkpoint_common(True) def _test_add_to_checkpoint_common(self, temporary): - files = set(("foo.bar", "baz.qux",)) + files = {"foo.bar", "baz.qux",} save_notes = "foo bar baz qux" installer_func = functools.partial(self.installer.add_to_checkpoint, @@ -242,17 +242,17 @@ class AddrTest(unittest.TestCase): def test_set_inclusion(self): from certbot.plugins.common import Addr - set_a = set([self.addr1, self.addr2]) + set_a = {self.addr1, self.addr2} addr1b = Addr.fromstring("192.168.1.1") addr2b = Addr.fromstring("192.168.1.1:*") - set_b = set([addr1b, addr2b]) + set_b = {addr1b, addr2b} self.assertEqual(set_a, set_b) - set_c = set([self.addr4, self.addr5]) + set_c = {self.addr4, self.addr5} addr4b = Addr.fromstring("[fe00::1]") addr5b = Addr.fromstring("[fe00::1]:*") - set_d = set([addr4b, addr5b]) + set_d = {addr4b, addr5b} self.assertEqual(set_c, set_d) diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 701abe109..862accb92 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -161,7 +161,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup(["chall1"]) self.assertEqual(self.auth.served, { - "server1": set(), "server2": set(["chall2", "chall3"])}) + "server1": set(), "server2": {"chall2", "chall3"}}) self.auth.servers.stop.assert_called_once_with(1) self.auth.servers.running.return_value = { @@ -169,12 +169,12 @@ class AuthenticatorTest(unittest.TestCase): } self.auth.cleanup(["chall2"]) self.assertEqual(self.auth.served, { - "server1": set(), "server2": set(["chall3"])}) + "server1": set(), "server2": {"chall3"}}) self.assertEqual(1, self.auth.servers.stop.call_count) self.auth.cleanup(["chall3"]) self.assertEqual(self.auth.served, { - "server1": set(), "server2": set([])}) + "server1": set(), "server2": set()}) self.auth.servers.stop.assert_called_with(2) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index fc9133c5f..e592bcbdc 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -87,7 +87,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): # Check to make sure new files are also checked... self.assertRaises(errors.ReverterError, self.reverter.add_to_checkpoint, - set([config3]), "invalid save") + {config3}, "invalid save") def test_multiple_saves_and_temp_revert(self): self.reverter.add_to_temp_checkpoint(self.sets[0], "save1") @@ -414,9 +414,9 @@ def setup_test_files(): with open(config2, "w") as file_fd: file_fd.write("directive-dir2") - sets = [set([config1]), - set([config2]), - set([config1, config2])] + sets = [{config1}, + {config2}, + {config1, config2}] return config1, config2, dir1, dir2, sets diff --git a/letsencrypt-auto-source/rebuild_dependencies.py b/letsencrypt-auto-source/rebuild_dependencies.py old mode 100755 new mode 100644 index 6d1ec15ff..3093b6bb0 --- a/letsencrypt-auto-source/rebuild_dependencies.py +++ b/letsencrypt-auto-source/rebuild_dependencies.py @@ -103,7 +103,7 @@ def _requirements_from_one_distribution(distribution, verbose): os.chmod(script, 0o755) _write_to(authoritative_constraints, '\n'.join( - ['{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items()])) + '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) command = ['docker', 'run', '--rm', '--cidfile', cid_file, '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 75d0d5d33..5196cf9c4 100644 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -124,7 +124,9 @@ def _check_version(version_str, major_version): return False -def subprocess_with_print(cmd, env=os.environ, shell=False): +def subprocess_with_print(cmd, env=None, shell=False): + if env is None: + env = os.environ print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) subprocess.check_call(cmd, env=env, shell=shell) diff --git a/tools/deactivate.py b/tools/deactivate.py index 10c9ecd35..214c0595c 100644 --- a/tools/deactivate.py +++ b/tools/deactivate.py @@ -17,7 +17,6 @@ import sys from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa import josepy as jose from acme import client as acme_client diff --git a/windows-installer/construct.py b/windows-installer/construct.py index f0724f5f4..4b05c926a 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -153,7 +153,7 @@ extra_preamble=pywin32_paths.py '''.format(certbot_version=certbot_version, installer_suffix='win_amd64' if PYTHON_BITNESS == 64 else 'win32', python_bitness=PYTHON_BITNESS, - python_version='.'.join([str(item) for item in PYTHON_VERSION]))) + python_version='.'.join(str(item) for item in PYTHON_VERSION))) return installer_cfg_path @@ -184,7 +184,7 @@ if __name__ == '__main__': if sys.version_info[:2] != PYTHON_VERSION[:2]: raise RuntimeError('This script must be run with Python {0}' - .format('.'.join([str(item) for item in PYTHON_VERSION[0:2]]))) + .format('.'.join(str(item) for item in PYTHON_VERSION[0:2]))) if struct.calcsize('P') * 8 != PYTHON_BITNESS: raise RuntimeError('This script must be run with a {0} bit version of Python.' From cd0acf5dcc94263706b08d7f45d741fbc07c9196 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 13 Apr 2020 14:32:22 -0700 Subject: [PATCH 73/92] Do not require mock in Python 3 in acme module (#7894) Part of #7886. This PR conditionally installs mock in acme/setup.py based on setuptools version and python version, when possible. It then updates acme tests to use unittest.mock when mock isn't available. * Conditionally install mock in acme * use unittest.mock when third-party mock isn't available in acme * error when trying to build wheels with old setuptools --- acme/setup.py | 12 +++++++++++- acme/tests/challenges_test.py | 5 ++++- acme/tests/client_test.py | 5 ++++- acme/tests/errors_test.py | 5 ++++- acme/tests/magic_typing_test.py | 5 ++++- acme/tests/messages_test.py | 5 ++++- acme/tests/standalone_test.py | 5 ++++- 7 files changed, 35 insertions(+), 7 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 0527b3fb5..356410efe 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -15,7 +17,6 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'mock', # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13.1', 'pyrfc3339', @@ -26,6 +27,15 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'pytest', 'pytest-xdist', diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index 433c7bb82..b6a3d1f83 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -3,7 +3,10 @@ import unittest import josepy as jose import OpenSSL -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import requests from six.moves.urllib import parse as urllib_parse diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 010974a32..444737e15 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -6,7 +6,10 @@ import json import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL import requests from six.moves import http_client # pylint: disable=import-error diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index c9c6f484f..660c122d3 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -1,7 +1,10 @@ """Tests for acme.errors.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock class BadNonceTest(unittest.TestCase): diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 60b4a5df4..43efe8eff 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock class MagicTypingTest(unittest.TestCase): diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index d36e2cc99..9c2955f7e 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -2,7 +2,10 @@ import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges import test_util diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 8face7c7b..7a0f5ddde 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -4,7 +4,10 @@ import threading import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import requests from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error From 77871ba71c05dead938d700fefcfd719646301a1 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 13 Apr 2020 14:33:23 -0700 Subject: [PATCH 74/92] Do not require mock in Python 3 in certbot module (#7895) Part of #7886. This PR conditionally installs mock in `certbot/setup.py` based on setuptools version and python version, when possible. It then updates `certbot` tests to use `unittest.mock` when `mock` isn't available. * Conditionally install mock in certbot * use unittest.mock when third-party mock isn't available in certbot * Add type:ignores because of https://github.com/python/mypy/issues/1153 * error when trying to build wheels with old setuptools --- certbot/certbot/plugins/dns_test_common.py | 5 ++++- certbot/certbot/plugins/dns_test_common_lexicon.py | 5 ++++- certbot/certbot/tests/util.py | 5 ++++- certbot/setup.py | 12 ++++++++++-- certbot/tests/account_test.py | 5 ++++- certbot/tests/auth_handler_test.py | 5 ++++- certbot/tests/cert_manager_test.py | 5 ++++- certbot/tests/cli_test.py | 5 ++++- certbot/tests/client_test.py | 5 ++++- certbot/tests/compat/filesystem_test.py | 5 ++++- certbot/tests/configuration_test.py | 5 ++++- certbot/tests/crypto_util_test.py | 5 ++++- certbot/tests/display/completer_test.py | 5 ++++- certbot/tests/display/ops_test.py | 5 ++++- certbot/tests/display/util_test.py | 5 ++++- certbot/tests/eff_test.py | 5 ++++- certbot/tests/error_handler_test.py | 5 ++++- certbot/tests/errors_test.py | 5 ++++- certbot/tests/hook_test.py | 5 ++++- certbot/tests/lock_test.py | 5 ++++- certbot/tests/log_test.py | 5 ++++- certbot/tests/main_test.py | 5 ++++- certbot/tests/ocsp_test.py | 5 ++++- certbot/tests/plugins/common_test.py | 5 ++++- certbot/tests/plugins/disco_test.py | 5 ++++- certbot/tests/plugins/dns_common_lexicon_test.py | 5 ++++- certbot/tests/plugins/dns_common_test.py | 5 ++++- certbot/tests/plugins/enhancements_test.py | 5 ++++- certbot/tests/plugins/manual_test.py | 5 ++++- certbot/tests/plugins/null_test.py | 5 ++++- certbot/tests/plugins/selection_test.py | 5 ++++- certbot/tests/plugins/standalone_test.py | 5 ++++- certbot/tests/plugins/storage_test.py | 5 ++++- certbot/tests/plugins/util_test.py | 5 ++++- certbot/tests/plugins/webroot_test.py | 5 ++++- certbot/tests/renewal_test.py | 5 ++++- certbot/tests/renewupdater_test.py | 5 ++++- certbot/tests/reporter_test.py | 5 ++++- certbot/tests/reverter_test.py | 5 ++++- certbot/tests/storage_test.py | 5 ++++- certbot/tests/util_test.py | 5 ++++- 41 files changed, 170 insertions(+), 42 deletions(-) diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index 9ef76c2c3..d5044d336 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -2,7 +2,10 @@ import configobj import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six from acme import challenges diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index c77d6da9e..1bef06042 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -1,7 +1,10 @@ """Base test class for DNS authenticators built on Lexicon.""" import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 8b28b1080..92f52a852 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -10,7 +10,10 @@ import unittest from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import pkg_resources import six diff --git a/certbot/setup.py b/certbot/setup.py index 514aec8c5..143e1a10a 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -47,7 +47,6 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', @@ -62,7 +61,8 @@ install_requires = [ # So this dependency is not added for old Linux distributions with old setuptools, # in order to allow these systems to build certbot from sources. pywin32_req = 'pywin32>=227' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py -if StrictVersion(setuptools_version) >= StrictVersion('36.2'): +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: install_requires.append(pywin32_req + " ; sys_platform == 'win32'") elif 'bdist_wheel' in sys.argv[1:]: raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' @@ -73,6 +73,14 @@ elif os.name == 'nt': # setuptools, pywin32 will not be specified as a dependency. install_requires.append(pywin32_req) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'coverage', 'ipdb', diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 4a6ed3e01..6c6e6c860 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -4,7 +4,10 @@ import json import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz from acme import messages diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 7ab3a2baa..6cd207b4b 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -3,7 +3,10 @@ import functools import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from acme import challenges diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index bea64f09c..d956fd04f 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -7,7 +7,10 @@ import tempfile import unittest import configobj -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot._internal import configuration diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 7d21f8bb8..592c40be7 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -4,7 +4,10 @@ import copy import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 3e0e5b212..cbc058c7a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -5,7 +5,10 @@ import tempfile import unittest from josepy import interfaces -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index fdfb1ffe9..1c2d2df0d 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -3,7 +3,10 @@ import contextlib import errno import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import util from certbot._internal import lock diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index d748b9bfb..1f8a0e803 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.configuration.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot._internal import constants diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index d52e3acdb..481d83894 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -2,7 +2,10 @@ import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL import zope.component diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index a183fd14f..0852ab175 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -7,7 +7,10 @@ import string import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from six.moves import reload_module # pylint: disable=import-error from certbot.compat import filesystem # pylint: disable=ungrouped-imports diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 327f1bcbe..a683e1d3d 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,7 +4,10 @@ import sys import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from acme import messages diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index eccfee7db..3e492e9ab 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -4,7 +4,10 @@ import socket import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from certbot import errors diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index cdd7908a3..c4a25da69 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.eff.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import requests from certbot._internal import constants diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 899dbc611..e5a95c3a8 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -4,7 +4,10 @@ import signal import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.compat import os diff --git a/certbot/tests/errors_test.py b/certbot/tests/errors_test.py index d6c829322..792868df0 100644 --- a/certbot/tests/errors_test.py +++ b/certbot/tests/errors_test.py @@ -1,7 +1,10 @@ """Tests for certbot.errors.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import messages from certbot import achallenges diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 3b7a94489..32081f9d0 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.hooks.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 5a48009fd..2f887d33e 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -3,7 +3,10 @@ import functools import multiprocessing import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot.compat import os diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 5b0918ce5..5cd287c2e 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -5,7 +5,10 @@ import sys import time import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import messages diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 92afc3fef..8113b2bc4 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -13,7 +13,10 @@ import traceback import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 9eb49e115..af54844cf 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -10,7 +10,10 @@ from cryptography.exceptions import InvalidSignature from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes # type: ignore -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz from certbot import errors diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 1f339a6f3..344e6312f 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -4,7 +4,10 @@ import shutil import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges from certbot import achallenges diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index eec0795e3..5a0a392b0 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -3,7 +3,10 @@ import functools import string import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pkg_resources import six import zope.interface diff --git a/certbot/tests/plugins/dns_common_lexicon_test.py b/certbot/tests/plugins/dns_common_lexicon_test.py index 986362ca9..a67430f3e 100644 --- a/certbot/tests/plugins/dns_common_lexicon_test.py +++ b/certbot/tests/plugins/dns_common_lexicon_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.plugins import dns_common_lexicon from certbot.plugins import dns_test_common_lexicon diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index eba3c89d6..993f3b461 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -4,7 +4,10 @@ import collections import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/plugins/enhancements_test.py b/certbot/tests/plugins/enhancements_test.py index 05fbc5028..a20a6864f 100644 --- a/certbot/tests/plugins/enhancements_test.py +++ b/certbot/tests/plugins/enhancements_test.py @@ -1,7 +1,10 @@ """Tests for new style enhancements""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot._internal.plugins import null from certbot.plugins import enhancements diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 6cdef148a..933c759d6 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import challenges diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index db0213813..47708e340 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.plugins.null.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index c66473ad1..e5e6db031 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from certbot import errors diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 862accb92..751b9d943 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -5,7 +5,10 @@ from socket import errno as socket_errors # type: ignore import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL.crypto # pylint: disable=unused-import import six diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index e9ca2007f..4b0d1da83 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -2,7 +2,10 @@ import json import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot.compat import filesystem diff --git a/certbot/tests/plugins/util_test.py b/certbot/tests/plugins/util_test.py index c41e55222..9387b4ae7 100644 --- a/certbot/tests/plugins/util_test.py +++ b/certbot/tests/plugins/util_test.py @@ -1,7 +1,10 @@ """Tests for certbot.plugins.util.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.compat import os diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index fade12bb1..e57e09eae 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -10,7 +10,10 @@ import tempfile import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import challenges diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index e92211ea2..1fc54b42e 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.renewal""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges from certbot import errors diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index c6f8f3713..b5ecddb5a 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -1,7 +1,10 @@ """Tests for renewal updater interfaces""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import interfaces from certbot._internal import main diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 3d7c80172..7d03f1821 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index e592bcbdc..d67aa431a 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -5,7 +5,10 @@ import shutil import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from certbot import errors diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 0f7620b78..5aa37824d 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -6,7 +6,10 @@ import stat import unittest import configobj -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz import six diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 3ff09a83f..6dd0f964c 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -4,7 +4,10 @@ import errno import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from six.moves import reload_module # pylint: disable=import-error From ff732bf97538fe1ab94589d4ddeff294026f3ddd Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 13 Apr 2020 17:09:24 -0700 Subject: [PATCH 75/92] Revert the last two mock PRs (#7903) * Revert "Do not require mock in Python 3 in certbot module (#7895)" This reverts commit 77871ba71c05dead938d700fefcfd719646301a1. * Revert "Do not require mock in Python 3 in acme module (#7894)" This reverts commit cd0acf5dcc94263706b08d7f45d741fbc07c9196. --- acme/setup.py | 12 +----------- acme/tests/challenges_test.py | 5 +---- acme/tests/client_test.py | 5 +---- acme/tests/errors_test.py | 5 +---- acme/tests/magic_typing_test.py | 5 +---- acme/tests/messages_test.py | 5 +---- acme/tests/standalone_test.py | 5 +---- certbot/certbot/plugins/dns_test_common.py | 5 +---- certbot/certbot/plugins/dns_test_common_lexicon.py | 5 +---- certbot/certbot/tests/util.py | 5 +---- certbot/setup.py | 12 ++---------- certbot/tests/account_test.py | 5 +---- certbot/tests/auth_handler_test.py | 5 +---- certbot/tests/cert_manager_test.py | 5 +---- certbot/tests/cli_test.py | 5 +---- certbot/tests/client_test.py | 5 +---- certbot/tests/compat/filesystem_test.py | 5 +---- certbot/tests/configuration_test.py | 5 +---- certbot/tests/crypto_util_test.py | 5 +---- certbot/tests/display/completer_test.py | 5 +---- certbot/tests/display/ops_test.py | 5 +---- certbot/tests/display/util_test.py | 5 +---- certbot/tests/eff_test.py | 5 +---- certbot/tests/error_handler_test.py | 5 +---- certbot/tests/errors_test.py | 5 +---- certbot/tests/hook_test.py | 5 +---- certbot/tests/lock_test.py | 5 +---- certbot/tests/log_test.py | 5 +---- certbot/tests/main_test.py | 5 +---- certbot/tests/ocsp_test.py | 5 +---- certbot/tests/plugins/common_test.py | 5 +---- certbot/tests/plugins/disco_test.py | 5 +---- certbot/tests/plugins/dns_common_lexicon_test.py | 5 +---- certbot/tests/plugins/dns_common_test.py | 5 +---- certbot/tests/plugins/enhancements_test.py | 5 +---- certbot/tests/plugins/manual_test.py | 5 +---- certbot/tests/plugins/null_test.py | 5 +---- certbot/tests/plugins/selection_test.py | 5 +---- certbot/tests/plugins/standalone_test.py | 5 +---- certbot/tests/plugins/storage_test.py | 5 +---- certbot/tests/plugins/util_test.py | 5 +---- certbot/tests/plugins/webroot_test.py | 5 +---- certbot/tests/renewal_test.py | 5 +---- certbot/tests/renewupdater_test.py | 5 +---- certbot/tests/reporter_test.py | 5 +---- certbot/tests/reverter_test.py | 5 +---- certbot/tests/storage_test.py | 5 +---- certbot/tests/util_test.py | 5 +---- 48 files changed, 49 insertions(+), 205 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 356410efe..0527b3fb5 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,7 +1,5 @@ -from distutils.version import StrictVersion import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -17,6 +15,7 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', + 'mock', # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13.1', 'pyrfc3339', @@ -27,15 +26,6 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] -setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - dev_extras = [ 'pytest', 'pytest-xdist', diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index b6a3d1f83..433c7bb82 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -3,10 +3,7 @@ import unittest import josepy as jose import OpenSSL -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import requests from six.moves.urllib import parse as urllib_parse diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 444737e15..010974a32 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -6,10 +6,7 @@ import json import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import OpenSSL import requests from six.moves import http_client # pylint: disable=import-error diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index 660c122d3..c9c6f484f 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -1,10 +1,7 @@ """Tests for acme.errors.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock class BadNonceTest(unittest.TestCase): diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 43efe8eff..60b4a5df4 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -2,10 +2,7 @@ import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock class MagicTypingTest(unittest.TestCase): diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index 9c2955f7e..d36e2cc99 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -2,10 +2,7 @@ import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from acme import challenges import test_util diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 7a0f5ddde..8face7c7b 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -4,10 +4,7 @@ import threading import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import requests from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index d5044d336..9ef76c2c3 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -2,10 +2,7 @@ import configobj import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore +import mock import six from acme import challenges diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index 1bef06042..c77d6da9e 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -1,10 +1,7 @@ """Base test class for DNS authenticators built on Lexicon.""" import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore +import mock from requests.exceptions import HTTPError from requests.exceptions import RequestException diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 92f52a852..8b28b1080 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -10,10 +10,7 @@ import unittest from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore +import mock import OpenSSL import pkg_resources import six diff --git a/certbot/setup.py b/certbot/setup.py index 143e1a10a..514aec8c5 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -47,6 +47,7 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', + 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', @@ -61,8 +62,7 @@ install_requires = [ # So this dependency is not added for old Linux distributions with old setuptools, # in order to allow these systems to build certbot from sources. pywin32_req = 'pywin32>=227' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py -setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) -if setuptools_known_environment_markers: +if StrictVersion(setuptools_version) >= StrictVersion('36.2'): install_requires.append(pywin32_req + " ; sys_platform == 'win32'") elif 'bdist_wheel' in sys.argv[1:]: raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' @@ -73,14 +73,6 @@ elif os.name == 'nt': # setuptools, pywin32 will not be specified as a dependency. install_requires.append(pywin32_req) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - dev_extras = [ 'coverage', 'ipdb', diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 6c6e6c860..4a6ed3e01 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -4,10 +4,7 @@ import json import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import pytz from acme import messages diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 6cd207b4b..7ab3a2baa 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -3,10 +3,7 @@ import functools import logging import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import zope.component from acme import challenges diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index d956fd04f..bea64f09c 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -7,10 +7,7 @@ import tempfile import unittest import configobj -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot._internal import configuration diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 592c40be7..7d21f8bb8 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -4,10 +4,7 @@ import copy import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index cbc058c7a..3e0e5b212 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -5,10 +5,7 @@ import tempfile import unittest from josepy import interfaces -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot import util diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index 1c2d2df0d..fdfb1ffe9 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -3,10 +3,7 @@ import contextlib import errno import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import util from certbot._internal import lock diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index 1f8a0e803..d748b9bfb 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -1,10 +1,7 @@ """Tests for certbot._internal.configuration.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot._internal import constants diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 481d83894..d52e3acdb 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -2,10 +2,7 @@ import logging import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import OpenSSL import zope.component diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 0852ab175..a183fd14f 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -7,10 +7,7 @@ import string import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from six.moves import reload_module # pylint: disable=import-error from certbot.compat import filesystem # pylint: disable=ungrouped-imports diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index a683e1d3d..327f1bcbe 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,10 +4,7 @@ import sys import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import zope.component from acme import messages diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 3e492e9ab..eccfee7db 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -4,10 +4,7 @@ import socket import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from certbot import errors diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index c4a25da69..cdd7908a3 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -1,10 +1,7 @@ """Tests for certbot._internal.eff.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import requests from certbot._internal import constants diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index e5a95c3a8..899dbc611 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -4,10 +4,7 @@ import signal import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot.compat import os diff --git a/certbot/tests/errors_test.py b/certbot/tests/errors_test.py index 792868df0..d6c829322 100644 --- a/certbot/tests/errors_test.py +++ b/certbot/tests/errors_test.py @@ -1,10 +1,7 @@ """Tests for certbot.errors.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from acme import messages from certbot import achallenges diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 32081f9d0..3b7a94489 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -1,10 +1,7 @@ """Tests for certbot._internal.hooks.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot import util diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 2f887d33e..5a48009fd 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -3,10 +3,7 @@ import functools import multiprocessing import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot.compat import os diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 5cd287c2e..5b0918ce5 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -5,10 +5,7 @@ import sys import time import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from acme import messages diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 8113b2bc4..92afc3fef 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -13,10 +13,7 @@ import traceback import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import pytz import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index af54844cf..9eb49e115 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -10,10 +10,7 @@ from cryptography.exceptions import InvalidSignature from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes # type: ignore -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import pytz from certbot import errors diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 344e6312f..1f339a6f3 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -4,10 +4,7 @@ import shutil import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from acme import challenges from certbot import achallenges diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index 5a0a392b0..eec0795e3 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -3,10 +3,7 @@ import functools import string import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import pkg_resources import six import zope.interface diff --git a/certbot/tests/plugins/dns_common_lexicon_test.py b/certbot/tests/plugins/dns_common_lexicon_test.py index a67430f3e..986362ca9 100644 --- a/certbot/tests/plugins/dns_common_lexicon_test.py +++ b/certbot/tests/plugins/dns_common_lexicon_test.py @@ -2,10 +2,7 @@ import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot.plugins import dns_common_lexicon from certbot.plugins import dns_test_common_lexicon diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index 993f3b461..eba3c89d6 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -4,10 +4,7 @@ import collections import logging import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot import util diff --git a/certbot/tests/plugins/enhancements_test.py b/certbot/tests/plugins/enhancements_test.py index a20a6864f..05fbc5028 100644 --- a/certbot/tests/plugins/enhancements_test.py +++ b/certbot/tests/plugins/enhancements_test.py @@ -1,10 +1,7 @@ """Tests for new style enhancements""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot._internal.plugins import null from certbot.plugins import enhancements diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 933c759d6..6cdef148a 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -2,10 +2,7 @@ import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from acme import challenges diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index 47708e340..db0213813 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -1,10 +1,7 @@ """Tests for certbot._internal.plugins.null.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index e5e6db031..c66473ad1 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -2,10 +2,7 @@ import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import zope.component from certbot import errors diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 751b9d943..862accb92 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -5,10 +5,7 @@ from socket import errno as socket_errors # type: ignore import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import OpenSSL.crypto # pylint: disable=unused-import import six diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index 4b0d1da83..e9ca2007f 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -2,10 +2,7 @@ import json import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import errors from certbot.compat import filesystem diff --git a/certbot/tests/plugins/util_test.py b/certbot/tests/plugins/util_test.py index 9387b4ae7..c41e55222 100644 --- a/certbot/tests/plugins/util_test.py +++ b/certbot/tests/plugins/util_test.py @@ -1,10 +1,7 @@ """Tests for certbot.plugins.util.""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot.compat import os diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index e57e09eae..fade12bb1 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -10,10 +10,7 @@ import tempfile import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from acme import challenges diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index 1fc54b42e..e92211ea2 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -1,10 +1,7 @@ """Tests for certbot._internal.renewal""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from acme import challenges from certbot import errors diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index b5ecddb5a..c6f8f3713 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -1,10 +1,7 @@ """Tests for renewal updater interfaces""" import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock from certbot import interfaces from certbot._internal import main diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 7d03f1821..3d7c80172 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -2,10 +2,7 @@ import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index d67aa431a..e592bcbdc 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -5,10 +5,7 @@ import shutil import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from certbot import errors diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 5aa37824d..0f7620b78 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -6,10 +6,7 @@ import stat import unittest import configobj -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import pytz import six diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 6dd0f964c..3ff09a83f 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -4,10 +4,7 @@ import errno import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock +import mock import six from six.moves import reload_module # pylint: disable=import-error From 569df2d37a02f95096f0004369491ac24f8630e6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 14 Apr 2020 17:01:59 -0700 Subject: [PATCH 76/92] Remove Ubuntu 19.04 tests. (#7906) This PR fixes the Travis failures that can be seen https://travis-ci.com/certbot/certbot/builds/160258644. Running the tests locally, it looks like Ubuntu has started shutting down the 19.04 repos which makes sense as this release has been EOL'd. See https://wiki.ubuntu.com/Releases. I have the full suite including the test farm tests running at https://travis-ci.com/github/certbot/certbot/builds/160269969 with this change. The issue of adding 19.10 to our test farm tests is tracked by #7851. I think that issue is important and it's in our current milestone, but I'd personally rather get our tests passing for now and try to expand them to run on other systems later. --- tests/letstest/apache2_targets.yaml | 5 ----- tests/letstest/targets.yaml | 5 ----- 2 files changed, 10 deletions(-) diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml index 1450a8578..76415b27e 100644 --- a/tests/letstest/apache2_targets.yaml +++ b/tests/letstest/apache2_targets.yaml @@ -1,11 +1,6 @@ targets: #----------------------------------------------------------------------------- #Ubuntu - - ami: ami-08ab45c4343f5f5c6 - name: ubuntu19.04 - type: ubuntu - virt: hvm - user: ubuntu - ami: ami-095192256fe1477ad name: ubuntu18.04LTS type: ubuntu diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 188be8e24..06055a9a5 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -1,11 +1,6 @@ targets: #----------------------------------------------------------------------------- #Ubuntu - - ami: ami-08ab45c4343f5f5c6 - name: ubuntu19.04 - type: ubuntu - virt: hvm - user: ubuntu - ami: ami-095192256fe1477ad name: ubuntu18.04LTS type: ubuntu From 127d2dc307fe827695a8ec69a373687b67c75dae Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:27:55 -0700 Subject: [PATCH 77/92] Do not require mock in Python 3 in acme module (#7910) Part of #7886. This PR conditionally installs mock in `acme/setup.py` based on setuptools version and python version, when possible. It then updates `acme` tests to use `unittest.mock` when `mock` isn't available. Now with `type: ignore` as appropriate. Once the "future steps" of #7886 are finished, and mypy is on Python 3, the `pragma no cover`s and `type ignore`s will be gone. * Conditionally install mock in acme * error out on newer python and older setuptools * error when trying to build wheels with old setuptools * use unittest.mock when third-party mock isn't available in acme, with no cover and type ignore --- acme/setup.py | 12 +++++++++++- acme/tests/challenges_test.py | 5 ++++- acme/tests/client_test.py | 5 ++++- acme/tests/errors_test.py | 5 ++++- acme/tests/magic_typing_test.py | 5 ++++- acme/tests/messages_test.py | 5 ++++- acme/tests/standalone_test.py | 5 ++++- 7 files changed, 35 insertions(+), 7 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 0527b3fb5..356410efe 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -15,7 +17,6 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'mock', # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13.1', 'pyrfc3339', @@ -26,6 +27,15 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'pytest', 'pytest-xdist', diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index 433c7bb82..70371051c 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -3,7 +3,10 @@ import unittest import josepy as jose import OpenSSL -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import requests from six.moves.urllib import parse as urllib_parse diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 010974a32..c90cad9b0 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -6,7 +6,10 @@ import json import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import requests from six.moves import http_client # pylint: disable=import-error diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index c9c6f484f..fb90a3f0d 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -1,7 +1,10 @@ """Tests for acme.errors.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore class BadNonceTest(unittest.TestCase): diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 60b4a5df4..9e4fd29f5 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore class MagicTypingTest(unittest.TestCase): diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index d36e2cc99..890a5f413 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -2,7 +2,10 @@ import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from acme import challenges import test_util diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 8face7c7b..d03b56535 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -4,7 +4,10 @@ import threading import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import requests from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error From 35fb99b86f4802694acfdefe5bcc5ba1d28e70c6 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:28:47 -0700 Subject: [PATCH 78/92] Do not require mock in Python 3 in certbot module (#7911) This PR is exactly the same as #7895, but know we know a little bit more about what was going on with `mypy`. Part of #7886. This PR conditionally installs mock in `certbot/setup.py` based on setuptools version and python version, when possible. It then updates `certbot` tests to use `unittest.mock` when `mock` isn't available. * Conditionally install mock in certbot * use unittest.mock when third-party mock isn't available in certbot * Add type:ignores because of https://github.com/python/mypy/issues/1153 * error out on newer python and older setuptools * error when trying to build wheels with old setuptools --- certbot/certbot/plugins/dns_test_common.py | 5 ++++- certbot/certbot/plugins/dns_test_common_lexicon.py | 5 ++++- certbot/certbot/tests/util.py | 5 ++++- certbot/setup.py | 12 ++++++++++-- certbot/tests/account_test.py | 5 ++++- certbot/tests/auth_handler_test.py | 5 ++++- certbot/tests/cert_manager_test.py | 5 ++++- certbot/tests/cli_test.py | 5 ++++- certbot/tests/client_test.py | 5 ++++- certbot/tests/compat/filesystem_test.py | 5 ++++- certbot/tests/configuration_test.py | 5 ++++- certbot/tests/crypto_util_test.py | 5 ++++- certbot/tests/display/completer_test.py | 5 ++++- certbot/tests/display/ops_test.py | 5 ++++- certbot/tests/display/util_test.py | 5 ++++- certbot/tests/eff_test.py | 5 ++++- certbot/tests/error_handler_test.py | 5 ++++- certbot/tests/errors_test.py | 5 ++++- certbot/tests/hook_test.py | 5 ++++- certbot/tests/lock_test.py | 5 ++++- certbot/tests/log_test.py | 5 ++++- certbot/tests/main_test.py | 5 ++++- certbot/tests/ocsp_test.py | 5 ++++- certbot/tests/plugins/common_test.py | 5 ++++- certbot/tests/plugins/disco_test.py | 5 ++++- certbot/tests/plugins/dns_common_lexicon_test.py | 5 ++++- certbot/tests/plugins/dns_common_test.py | 5 ++++- certbot/tests/plugins/enhancements_test.py | 5 ++++- certbot/tests/plugins/manual_test.py | 5 ++++- certbot/tests/plugins/null_test.py | 5 ++++- certbot/tests/plugins/selection_test.py | 5 ++++- certbot/tests/plugins/standalone_test.py | 5 ++++- certbot/tests/plugins/storage_test.py | 5 ++++- certbot/tests/plugins/util_test.py | 5 ++++- certbot/tests/plugins/webroot_test.py | 5 ++++- certbot/tests/renewal_test.py | 5 ++++- certbot/tests/renewupdater_test.py | 5 ++++- certbot/tests/reporter_test.py | 5 ++++- certbot/tests/reverter_test.py | 5 ++++- certbot/tests/storage_test.py | 5 ++++- certbot/tests/util_test.py | 5 ++++- 41 files changed, 170 insertions(+), 42 deletions(-) diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index 9ef76c2c3..d5044d336 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -2,7 +2,10 @@ import configobj import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six from acme import challenges diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index c77d6da9e..1bef06042 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -1,7 +1,10 @@ """Base test class for DNS authenticators built on Lexicon.""" import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 8b28b1080..92f52a852 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -10,7 +10,10 @@ import unittest from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import pkg_resources import six diff --git a/certbot/setup.py b/certbot/setup.py index 514aec8c5..143e1a10a 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -47,7 +47,6 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', @@ -62,7 +61,8 @@ install_requires = [ # So this dependency is not added for old Linux distributions with old setuptools, # in order to allow these systems to build certbot from sources. pywin32_req = 'pywin32>=227' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py -if StrictVersion(setuptools_version) >= StrictVersion('36.2'): +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: install_requires.append(pywin32_req + " ; sys_platform == 'win32'") elif 'bdist_wheel' in sys.argv[1:]: raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' @@ -73,6 +73,14 @@ elif os.name == 'nt': # setuptools, pywin32 will not be specified as a dependency. install_requires.append(pywin32_req) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'coverage', 'ipdb', diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 4a6ed3e01..6c6e6c860 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -4,7 +4,10 @@ import json import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz from acme import messages diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 7ab3a2baa..6cd207b4b 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -3,7 +3,10 @@ import functools import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from acme import challenges diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index bea64f09c..d956fd04f 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -7,7 +7,10 @@ import tempfile import unittest import configobj -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot._internal import configuration diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 7d21f8bb8..592c40be7 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -4,7 +4,10 @@ import copy import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 3e0e5b212..cbc058c7a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -5,7 +5,10 @@ import tempfile import unittest from josepy import interfaces -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index fdfb1ffe9..1c2d2df0d 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -3,7 +3,10 @@ import contextlib import errno import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import util from certbot._internal import lock diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index d748b9bfb..1f8a0e803 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.configuration.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot._internal import constants diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index d52e3acdb..481d83894 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -2,7 +2,10 @@ import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL import zope.component diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index a183fd14f..0852ab175 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -7,7 +7,10 @@ import string import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from six.moves import reload_module # pylint: disable=import-error from certbot.compat import filesystem # pylint: disable=ungrouped-imports diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 327f1bcbe..a683e1d3d 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,7 +4,10 @@ import sys import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from acme import messages diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index eccfee7db..3e492e9ab 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -4,7 +4,10 @@ import socket import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from certbot import errors diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index cdd7908a3..c4a25da69 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.eff.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import requests from certbot._internal import constants diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 899dbc611..e5a95c3a8 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -4,7 +4,10 @@ import signal import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.compat import os diff --git a/certbot/tests/errors_test.py b/certbot/tests/errors_test.py index d6c829322..792868df0 100644 --- a/certbot/tests/errors_test.py +++ b/certbot/tests/errors_test.py @@ -1,7 +1,10 @@ """Tests for certbot.errors.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import messages from certbot import achallenges diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 3b7a94489..32081f9d0 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.hooks.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 5a48009fd..2f887d33e 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -3,7 +3,10 @@ import functools import multiprocessing import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot.compat import os diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 5b0918ce5..5cd287c2e 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -5,7 +5,10 @@ import sys import time import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import messages diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 92afc3fef..8113b2bc4 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -13,7 +13,10 @@ import traceback import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz import six from six.moves import reload_module # pylint: disable=import-error diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 9eb49e115..af54844cf 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -10,7 +10,10 @@ from cryptography.exceptions import InvalidSignature from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes # type: ignore -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz from certbot import errors diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 1f339a6f3..344e6312f 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -4,7 +4,10 @@ import shutil import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges from certbot import achallenges diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index eec0795e3..5a0a392b0 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -3,7 +3,10 @@ import functools import string import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pkg_resources import six import zope.interface diff --git a/certbot/tests/plugins/dns_common_lexicon_test.py b/certbot/tests/plugins/dns_common_lexicon_test.py index 986362ca9..a67430f3e 100644 --- a/certbot/tests/plugins/dns_common_lexicon_test.py +++ b/certbot/tests/plugins/dns_common_lexicon_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.plugins import dns_common_lexicon from certbot.plugins import dns_test_common_lexicon diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index eba3c89d6..993f3b461 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -4,7 +4,10 @@ import collections import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/plugins/enhancements_test.py b/certbot/tests/plugins/enhancements_test.py index 05fbc5028..a20a6864f 100644 --- a/certbot/tests/plugins/enhancements_test.py +++ b/certbot/tests/plugins/enhancements_test.py @@ -1,7 +1,10 @@ """Tests for new style enhancements""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot._internal.plugins import null from certbot.plugins import enhancements diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 6cdef148a..933c759d6 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import challenges diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index db0213813..47708e340 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.plugins.null.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index c66473ad1..e5e6db031 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from certbot import errors diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 862accb92..751b9d943 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -5,7 +5,10 @@ from socket import errno as socket_errors # type: ignore import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL.crypto # pylint: disable=unused-import import six diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index e9ca2007f..4b0d1da83 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -2,7 +2,10 @@ import json import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot.compat import filesystem diff --git a/certbot/tests/plugins/util_test.py b/certbot/tests/plugins/util_test.py index c41e55222..9387b4ae7 100644 --- a/certbot/tests/plugins/util_test.py +++ b/certbot/tests/plugins/util_test.py @@ -1,7 +1,10 @@ """Tests for certbot.plugins.util.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.compat import os diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index fade12bb1..e57e09eae 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -10,7 +10,10 @@ import tempfile import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import challenges diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index e92211ea2..1fc54b42e 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.renewal""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges from certbot import errors diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index c6f8f3713..b5ecddb5a 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -1,7 +1,10 @@ """Tests for renewal updater interfaces""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import interfaces from certbot._internal import main diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 3d7c80172..7d03f1821 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index e592bcbdc..d67aa431a 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -5,7 +5,10 @@ import shutil import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from certbot import errors diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 0f7620b78..5aa37824d 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -6,7 +6,10 @@ import stat import unittest import configobj -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz import six diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 3ff09a83f..6dd0f964c 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -4,7 +4,10 @@ import errno import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from six.moves import reload_module # pylint: disable=import-error From 8fb9a395ab4393c5d06704de01687421f7acee1f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:30:08 -0700 Subject: [PATCH 79/92] Do not require mock in Python 3 in apache module (#7896) Part of #7886. This PR conditionally installs mock in `apache/setup.py` based on setuptools version and python version, when possible. It then updates `apache` tests to use `unittest.mock` when `mock` isn't available. * Conditionally install mock in apache * error out on newer python and older setuptools * error when trying to build wheels with old setuptools * use unittest.mock when third-party mock isn't available in apache, with no cover and type ignore --- certbot-apache/local-oldest-requirements.txt | 2 +- certbot-apache/setup.py | 12 +++++++++++- certbot-apache/tests/augeasnode_test.py | 5 ++++- certbot-apache/tests/autohsts_test.py | 5 ++++- certbot-apache/tests/centos_test.py | 5 ++++- certbot-apache/tests/configurator_reverter_test.py | 5 ++++- certbot-apache/tests/configurator_test.py | 5 ++++- certbot-apache/tests/debian_test.py | 5 ++++- certbot-apache/tests/display_ops_test.py | 5 ++++- certbot-apache/tests/dualnode_test.py | 5 ++++- certbot-apache/tests/entrypoint_test.py | 5 ++++- certbot-apache/tests/fedora_test.py | 5 ++++- certbot-apache/tests/gentoo_test.py | 5 ++++- certbot-apache/tests/http_01_test.py | 5 ++++- certbot-apache/tests/parser_test.py | 5 ++++- certbot-apache/tests/parsernode_configurator_test.py | 5 ++++- certbot-apache/tests/util.py | 5 ++++- 17 files changed, 72 insertions(+), 17 deletions(-) diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt index cf61c15a5..e45a0f831 100644 --- a/certbot-apache/local-oldest-requirements.txt +++ b/certbot-apache/local-oldest-requirements.txt @@ -1,3 +1,3 @@ # Remember to update setup.py to match the package versions below. acme[dev]==0.29.0 -certbot[dev]==1.1.0 +certbot[dev]==1.1.0 \ No newline at end of file diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 14300370a..25fb920c2 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,13 +13,21 @@ version = '1.4.0.dev0' install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', - 'mock', 'python-augeas', 'setuptools', 'zope.component', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'apacheconfig>=0.3.2', ] diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 270cc8c44..abe72a5d0 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -1,5 +1,8 @@ """Tests for AugeasParserNode classes""" -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import os import util diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index 8e4f15d6b..d15600215 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -3,7 +3,10 @@ import re import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six # pylint: disable=unused-import # six is used in mock.patch() from certbot import errors diff --git a/certbot-apache/tests/centos_test.py b/certbot-apache/tests/centos_test.py index 40d1361d4..9dc6fa5a7 100644 --- a/certbot-apache/tests/centos_test.py +++ b/certbot-apache/tests/centos_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.configurator for Centos overrides""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import filesystem diff --git a/certbot-apache/tests/configurator_reverter_test.py b/certbot-apache/tests/configurator_reverter_test.py index ad8e73347..8596195d8 100644 --- a/certbot-apache/tests/configurator_reverter_test.py +++ b/certbot-apache/tests/configurator_reverter_test.py @@ -2,7 +2,10 @@ import shutil import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors import util diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 694c1c0bb..8fd3cb750 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -6,7 +6,10 @@ import socket import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six # pylint: disable=unused-import # six is used in mock.patch() from acme import challenges diff --git a/certbot-apache/tests/debian_test.py b/certbot-apache/tests/debian_test.py index 7a65bb7b1..192e3cba5 100644 --- a/certbot-apache/tests/debian_test.py +++ b/certbot-apache/tests/debian_test.py @@ -2,7 +2,10 @@ import shutil import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-apache/tests/display_ops_test.py b/certbot-apache/tests/display_ops_test.py index e9f54a562..4559668ac 100644 --- a/certbot-apache/tests/display_ops_test.py +++ b/certbot-apache/tests/display_ops_test.py @@ -1,7 +1,10 @@ """Test certbot_apache._internal.display_ops.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.display import util as display_util diff --git a/certbot-apache/tests/dualnode_test.py b/certbot-apache/tests/dualnode_test.py index 0871bac78..44cc69ff4 100644 --- a/certbot-apache/tests/dualnode_test.py +++ b/certbot-apache/tests/dualnode_test.py @@ -1,7 +1,10 @@ """Tests for DualParserNode implementation""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot_apache._internal import assertions from certbot_apache._internal import augeasparser diff --git a/certbot-apache/tests/entrypoint_test.py b/certbot-apache/tests/entrypoint_test.py index 04c393bdf..6f6f5bbb0 100644 --- a/certbot-apache/tests/entrypoint_test.py +++ b/certbot-apache/tests/entrypoint_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.entrypoint for override class resolution""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot_apache._internal import configurator from certbot_apache._internal import entrypoint diff --git a/certbot-apache/tests/fedora_test.py b/certbot-apache/tests/fedora_test.py index 7f1d6526f..e0ee603c3 100644 --- a/certbot-apache/tests/fedora_test.py +++ b/certbot-apache/tests/fedora_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.configurator for Fedora 29+ overrides""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import filesystem diff --git a/certbot-apache/tests/gentoo_test.py b/certbot-apache/tests/gentoo_test.py index 5a35c17ae..aa923c367 100644 --- a/certbot-apache/tests/gentoo_test.py +++ b/certbot-apache/tests/gentoo_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.configurator for Gentoo overrides""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import filesystem diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 7d9019702..696cd4a54 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -2,7 +2,10 @@ import unittest import errno -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from acme import challenges from certbot import achallenges diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py index 299eb4567..7aedec31d 100644 --- a/certbot-apache/tests/parser_test.py +++ b/certbot-apache/tests/parser_test.py @@ -2,7 +2,10 @@ import shutil import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-apache/tests/parsernode_configurator_test.py b/certbot-apache/tests/parsernode_configurator_test.py index afaaa6e43..7fbec2540 100644 --- a/certbot-apache/tests/parsernode_configurator_test.py +++ b/certbot-apache/tests/parsernode_configurator_test.py @@ -1,7 +1,10 @@ """Tests for ApacheConfigurator for AugeasParserNode classes""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import util diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index a39e5225d..f2a6a0263 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -5,7 +5,10 @@ import unittest import augeas import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import zope.component from certbot.compat import os From 49912732aca0903915c95011d9602423e1174c8f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:39:44 -0700 Subject: [PATCH 80/92] Do not require mock in Python 3 in nginx module (#7898) * Do not require mock in Python 3 in nginx module * error when trying to build wheels with old setuptools * add type: ignores --- certbot-nginx/setup.py | 12 +++++++++++- certbot-nginx/tests/configurator_test.py | 5 ++++- certbot-nginx/tests/http_01_test.py | 5 ++++- certbot-nginx/tests/parser_obj_test.py | 5 ++++- certbot-nginx/tests/test_util.py | 5 ++++- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 0d62e7d55..0e6deceb3 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,13 +13,21 @@ version = '1.4.0.dev0' install_requires = [ 'acme>=1.4.0.dev0', 'certbot>=1.4.0.dev0', - 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + class PyTest(TestCommand): user_options = [] diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 0a04a22a4..2c3264a5f 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -1,7 +1,10 @@ """Test for certbot_nginx._internal.configurator.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL from acme import challenges diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index 6418a8841..8f0673c1f 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -2,7 +2,10 @@ import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six from acme import challenges diff --git a/certbot-nginx/tests/parser_obj_test.py b/certbot-nginx/tests/parser_obj_test.py index bb7834701..8262c5f52 100644 --- a/certbot-nginx/tests/parser_obj_test.py +++ b/certbot-nginx/tests/parser_obj_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot_nginx._internal.parser_obj import COMMENT_BLOCK from certbot_nginx._internal.parser_obj import parse_raw diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index 4c9da84bd..4b26f7935 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -4,7 +4,10 @@ import shutil import tempfile import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import pkg_resources import zope.component From af21d1d56ece276e4a259ceba894e4d008446827 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:40:03 -0700 Subject: [PATCH 81/92] Do not require mock in Python 3 in certbot-compatibility-test module (#7899) * Do not require mock in Python 3 in certbot-compatibility-test module * error when trying to build wheels with old setuptools * add type: ignores --- .../configurators/apache/common.py | 5 ++++- .../certbot_compatibility_test/validator_test.py | 5 ++++- certbot-compatibility-test/setup.py | 12 +++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index a9b1ce87e..5d5542ffd 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -3,7 +3,10 @@ import os import shutil import subprocess -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import zope.interface from certbot import errors as le_errors diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator_test.py b/certbot-compatibility-test/certbot_compatibility_test/validator_test.py index 235ce0e3c..0b1056561 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator_test.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator_test.py @@ -1,7 +1,10 @@ """Tests for certbot_compatibility_test.validator.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import requests diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index d6760576a..ed8e3f861 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup @@ -8,12 +10,20 @@ version = '1.4.0.dev0' install_requires = [ 'certbot', 'certbot-apache', - 'mock', 'six', 'requests', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + if sys.version_info < (2, 7, 9): # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) install_requires.append('ndg-httpsclient') From 9c345ac301287fe8f3222a1084aa79a0dbca13e3 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 15 Apr 2020 11:54:44 -0700 Subject: [PATCH 82/92] Do not require mock in Python 3 in certbot-dns modules (#7900) Part of #7886. This PR conditionally installs `mock` in `certbot-dns-*/setup.py` based on setuptools version and python version, when possible. It then updates the tests to use `unittest.mock` when `mock` isn't available. * Do not require mock in Python 3 in certbot-dns modules * update changelog * error when trying to build wheels with old setuptools * add type: ignores --- certbot-dns-cloudflare/setup.py | 12 +++++++++++- certbot-dns-cloudflare/tests/dns_cloudflare_test.py | 5 ++++- certbot-dns-cloudxns/setup.py | 12 +++++++++++- certbot-dns-cloudxns/tests/dns_cloudxns_test.py | 5 ++++- certbot-dns-digitalocean/setup.py | 12 +++++++++++- .../tests/dns_digitalocean_test.py | 5 ++++- certbot-dns-dnsimple/setup.py | 12 +++++++++++- certbot-dns-dnsimple/tests/dns_dnsimple_test.py | 5 ++++- certbot-dns-dnsmadeeasy/setup.py | 12 +++++++++++- .../tests/dns_dnsmadeeasy_test.py | 5 ++++- certbot-dns-gehirn/setup.py | 12 +++++++++++- certbot-dns-gehirn/tests/dns_gehirn_test.py | 5 ++++- certbot-dns-google/setup.py | 12 +++++++++++- certbot-dns-google/tests/dns_google_test.py | 5 ++++- certbot-dns-linode/setup.py | 12 +++++++++++- certbot-dns-linode/tests/dns_linode_test.py | 5 ++++- certbot-dns-luadns/setup.py | 12 +++++++++++- certbot-dns-luadns/tests/dns_luadns_test.py | 5 ++++- certbot-dns-nsone/setup.py | 12 +++++++++++- certbot-dns-nsone/tests/dns_nsone_test.py | 5 ++++- certbot-dns-ovh/setup.py | 12 +++++++++++- certbot-dns-ovh/tests/dns_ovh_test.py | 5 ++++- certbot-dns-rfc2136/setup.py | 12 +++++++++++- certbot-dns-rfc2136/tests/dns_rfc2136_test.py | 5 ++++- certbot-dns-route53/setup.py | 12 +++++++++++- certbot-dns-route53/tests/dns_route53_test.py | 5 ++++- certbot-dns-sakuracloud/setup.py | 12 +++++++++++- .../tests/dns_sakuracloud_test.py | 5 ++++- certbot/CHANGELOG.md | 1 + 29 files changed, 211 insertions(+), 28 deletions(-) diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 67aac3231..9ac16bc67 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'cloudflare>=1.5.1', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py index d38330191..4d2dcf4ca 100644 --- a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py +++ b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py @@ -3,7 +3,10 @@ import unittest import CloudFlare -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 6c653967d..f998027f9 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py index a1e3cde89..43c69790f 100644 --- a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py +++ b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index c45fc8d03..7aef67d75 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,13 +13,21 @@ version = '1.4.0.dev0' install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', - 'mock', 'python-digitalocean>=1.11', 'setuptools', 'six', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py index 71301a47c..a752f52d0 100644 --- a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py @@ -3,7 +3,10 @@ import unittest import digitalocean -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 9124f0552..4a3f863f5 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -1,6 +1,8 @@ +from distutils.version import StrictVersion import os import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ version = '1.4.0.dev0' install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + # This package normally depends on dns-lexicon>=3.2.1 to address the # problem described in https://github.com/AnalogJ/lexicon/issues/387, # however, the fix there has been backported to older versions of diff --git a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py index ca5eb4f36..40eba4754 100644 --- a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py +++ b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2a4fd92b0..2dced23bf 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py index b94cc7d05..4a69e977c 100644 --- a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py +++ b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 2198fdd3e..7c7acc503 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,11 +13,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.1.22', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-gehirn/tests/dns_gehirn_test.py b/certbot-dns-gehirn/tests/dns_gehirn_test.py index f5b95b6c3..0598a5eb5 100644 --- a/certbot-dns-gehirn/tests/dns_gehirn_test.py +++ b/certbot-dns-gehirn/tests/dns_gehirn_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 087766edd..1b51a781e 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,7 +14,6 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'google-api-python-client>=1.5.5', - 'mock', 'oauth2client>=4.0', 'setuptools', 'zope.interface', @@ -20,6 +21,15 @@ install_requires = [ 'httplib2' ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 647a75b05..5af027cef 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -6,7 +6,10 @@ from googleapiclient import discovery from googleapiclient.errors import Error from googleapiclient.http import HttpMock from httplib2 import ServerNotFoundError -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 1e6b96b71..860c40079 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,11 +13,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.3', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-linode/tests/dns_linode_test.py b/certbot-dns-linode/tests/dns_linode_test.py index 3cf615486..fb9b1aa93 100644 --- a/certbot-dns-linode/tests/dns_linode_test.py +++ b/certbot-dns-linode/tests/dns_linode_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4db50b56c..83932e140 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-luadns/tests/dns_luadns_test.py b/certbot-dns-luadns/tests/dns_luadns_test.py index 934d3e103..a1242582f 100644 --- a/certbot-dns-luadns/tests/dns_luadns_test.py +++ b/certbot-dns-luadns/tests/dns_luadns_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 49e5e3bcf..459c6d752 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-nsone/tests/dns_nsone_test.py b/certbot-dns-nsone/tests/dns_nsone_test.py index dd6168f08..83371252f 100644 --- a/certbot-dns-nsone/tests/dns_nsone_test.py +++ b/certbot-dns-nsone/tests/dns_nsone_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 6c66b39dc..5823b237e 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-ovh/tests/dns_ovh_test.py b/certbot-dns-ovh/tests/dns_ovh_test.py index a420239ab..dd0f3058b 100644 --- a/certbot-dns-ovh/tests/dns_ovh_test.py +++ b/certbot-dns-ovh/tests/dns_ovh_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 5e0900e4d..94cda6f65 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'dnspython', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py index c767dba23..4c14a8072 100644 --- a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py @@ -5,7 +5,10 @@ import unittest import dns.flags import dns.rcode import dns.tsig -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 98455c362..f140d3f8d 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'boto3', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + class PyTest(TestCommand): user_options = [] diff --git a/certbot-dns-route53/tests/dns_route53_test.py b/certbot-dns-route53/tests/dns_route53_test.py index 85ec259b1..a77495313 100644 --- a/certbot-dns-route53/tests/dns_route53_test.py +++ b/certbot-dns-route53/tests/dns_route53_test.py @@ -4,7 +4,10 @@ import unittest from botocore.exceptions import ClientError from botocore.exceptions import NoCredentialsError -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 16990a4d3..b57e28577 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -1,5 +1,7 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand @@ -11,11 +13,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.1.23', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py index 16890b5a9..af94336b3 100644 --- a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py +++ b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index f61cdcfc7..1b9379de7 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -19,6 +19,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * Stop asking interactively if the user would like to add a redirect. +* `mock` dependency is now conditional on Python 2 in all of our packages. ### Fixed From f66314926aa842804b8b64af785f2cef5296e0c5 Mon Sep 17 00:00:00 2001 From: April King Date: Wed, 15 Apr 2020 15:54:17 -0500 Subject: [PATCH 83/92] Update URL for Mozilla SSL Configuration Generator (#7912) --- AUTHORS.md | 1 + certbot/docs/ciphers.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS.md b/AUTHORS.md index 4414076fc..21a6e7773 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -21,6 +21,7 @@ Authors * [Andrzej Górski](https://github.com/andrzej3393) * [Anselm Levskaya](https://github.com/levskaya) * [Antoine Jacoutot](https://github.com/ajacoutot) +* [April King](https://github.com/april) * [asaph](https://github.com/asaph) * [Axel Beckert](https://github.com/xtaran) * [Bas](https://github.com/Mechazawa) diff --git a/certbot/docs/ciphers.rst b/certbot/docs/ciphers.rst index 04b24b526..325d6244c 100644 --- a/certbot/docs/ciphers.rst +++ b/certbot/docs/ciphers.rst @@ -241,7 +241,7 @@ Mozilla Mozilla's general server configuration guidance is available at https://wiki.mozilla.org/Security/Server_Side_TLS -Mozilla has also produced a configuration generator: https://mozilla.github.io/server-side-tls/ssl-config-generator/ +Mozilla has also produced a configuration generator: https://ssl-config.mozilla.org Dutch National Cyber Security Centre ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 859dc38cb9195de072bc46e30e3edc0dab04f84d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 16 Apr 2020 08:59:40 -0700 Subject: [PATCH 84/92] Consolidate cover envs and default to py3-cover (#7905) * Consolidate cover envs and default to py3-cover * use py38 for code coverage in Travis * Disable coverage on Python < 3.6 line. --- .travis.yml | 10 ++++---- .../certbot_apache/_internal/parser.py | 2 +- tox.ini | 23 +++++-------------- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae25a6895..d3eeb1e03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,8 +44,8 @@ matrix: <<: *not-on-master # This job is always executed, including on master - - python: "2.7" - env: TOXENV=py27-cover FYI="py27 tests + code coverage" + - python: "3.8" + env: TOXENV=py38-cover FYI="py38 tests + code coverage" - python: "3.7" env: TOXENV=lint @@ -60,12 +60,12 @@ matrix: dist: trusty env: TOXENV='py27-{acme,apache,apache-v2,certbot,dns,nginx}-oldest' <<: *not-on-master + - python: "2.7" + env: TOXENV=py27 + <<: *not-on-master - python: "3.5" env: TOXENV=py35 <<: *not-on-master - - python: "3.8" - env: TOXENV=py38 - <<: *not-on-master - sudo: required env: TOXENV=apache_compat services: docker diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index f6aa3fe48..c9aebae54 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -741,7 +741,7 @@ class ApacheParser(object): """ if sys.version_info < (3, 6): # This strips off final /Z(?ms) - return fnmatch.translate(clean_fn_match)[:-7] + return fnmatch.translate(clean_fn_match)[:-7] # pragma: no cover # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z return fnmatch.translate(clean_fn_match)[4:-3] # pragma: no cover diff --git a/tox.ini b/tox.ini index 8aa4bfbf2..7f5b7bd5a 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py3,py27-cover,lint,mypy +envlist = modification,py3-cover,lint,mypy [base] # pip installs the requested packages in editable mode @@ -63,8 +63,11 @@ source_paths = passenv = CERTBOT_NO_PIN commands = - {[base]install_and_test} {[base]all_packages} - python tests/lock_test.py + !cover: {[base]install_and_test} {[base]all_packages} + !cover: python tests/lock_test.py + cover: {[base]install_packages} + cover: {[base]pip_install} certbot-apache[dev] + cover: python tox.cover.py # We always recreate the virtual environment to avoid problems like # https://github.com/certbot/certbot/issues/7745. recreate = true @@ -116,20 +119,6 @@ commands = setenv = {[testenv:py27-oldest]setenv} -[testenv:py27-cover] -basepython = python2.7 -commands = - {[base]install_packages} - {[base]pip_install} certbot-apache[dev] - python tox.cover.py - -[testenv:py37-cover] -basepython = python3.7 -commands = - {[base]install_packages} - {[base]pip_install} certbot-apache[dev] - python tox.cover.py - [testenv:lint] basepython = python3 # separating into multiple invocations disables cross package From 74eea40905c5305e7f1a90635a082574417df2b8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 16 Apr 2020 14:57:36 -0700 Subject: [PATCH 85/92] test on ubuntu 19.10 --- tests/letstest/apache2_targets.yaml | 5 +++++ tests/letstest/targets.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml index 76415b27e..1b92b9fdb 100644 --- a/tests/letstest/apache2_targets.yaml +++ b/tests/letstest/apache2_targets.yaml @@ -1,6 +1,11 @@ targets: #----------------------------------------------------------------------------- #Ubuntu + - ami: ami-0545f7036167eb3aa + name: ubuntu19.10 + type: ubuntu + virt: hvm + user: ubuntu - ami: ami-095192256fe1477ad name: ubuntu18.04LTS type: ubuntu diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 06055a9a5..04f611df8 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -1,6 +1,11 @@ targets: #----------------------------------------------------------------------------- #Ubuntu + - ami: ami-0545f7036167eb3aa + name: ubuntu19.10 + type: ubuntu + virt: hvm + user: ubuntu - ami: ami-095192256fe1477ad name: ubuntu18.04LTS type: ubuntu From 864ea08341ffaea8aebe70fc671da5e12eb2bc04 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 16 Apr 2020 15:00:28 -0700 Subject: [PATCH 86/92] test on fedora 31 --- tests/letstest/apache2_targets.yaml | 5 +++++ tests/letstest/targets.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml index 76415b27e..5885eed73 100644 --- a/tests/letstest/apache2_targets.yaml +++ b/tests/letstest/apache2_targets.yaml @@ -36,6 +36,11 @@ targets: user: admin #----------------------------------------------------------------------------- # Fedora + - ami: ami-0fcbe88944a53b4c8 + name: fedora31 + type: centos + virt: hvm + user: fedora - ami: ami-00bbc6858140f19ed name: fedora30 type: centos diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 06055a9a5..ce7be7d2c 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -50,6 +50,11 @@ targets: type: centos virt: hvm user: ec2-user + - ami: ami-0fcbe88944a53b4c8 + name: fedora31 + type: centos + virt: hvm + user: fedora - ami: ami-00bbc6858140f19ed name: fedora30 type: centos From 41306e1e375ecf8db2215cf90c7f21c51af5a406 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 20 Apr 2020 09:20:31 -0700 Subject: [PATCH 87/92] update rate limits url --- certbot/docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index d3c2d1582..1d8d54d3a 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -385,7 +385,7 @@ certificate exists alongside any previously obtained certificates, whether or not the previous certificates have expired. The generation of a new certificate counts against several rate limits that are intended to prevent abuse of the ACME protocol, as described -`here `__. +`here `__. .. _changing: From 5b749ff8f79303aa60d1dce166c40e7095c31876 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 20 Apr 2020 14:44:53 -0700 Subject: [PATCH 88/92] Use Python 3 in the release script. (#7918) Fixes #7902. --- tools/_release.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/_release.sh b/tools/_release.sh index 97d5f5eb8..7e483905e 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -59,7 +59,7 @@ mv "dist.$version" "dist.$version.$(date +%s).bak" || true git tag --delete "$tag" || true tmpvenv=$(mktemp -d) -VIRTUALENV_NO_DOWNLOAD=1 virtualenv -p python2 $tmpvenv +python3 -m venv "$tmpvenv" . $tmpvenv/bin/activate # update setuptools/pip just like in other places in the repo pip install -U setuptools @@ -157,7 +157,7 @@ done echo "Testing packages" cd "dist.$version" # start local PyPI -python -m SimpleHTTPServer $PORT & +python -m http.server $PORT & # cd .. is NOT done on purpose: we make sure that all subpackages are # installed from local PyPI rather than current directory (repo root) VIRTUALENV_NO_DOWNLOAD=1 virtualenv ../venv @@ -202,7 +202,7 @@ done # pin pip hashes of the things we just built for pkg in $SUBPKGS_IN_AUTO ; do echo $pkg==$version \\ - pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' + pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python -c 'from sys import stdin; input = stdin.read(); print(" ", input.replace("\n--hash", " \\\n --hash"), end="")' done > letsencrypt-auto-source/pieces/certbot-requirements.txt deactivate From 6693e87500344f0bff19f14beb2121f66a19d143 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 21 Apr 2020 10:27:48 -0700 Subject: [PATCH 89/92] Update dev docs to reflect Python 2 EOL. (#7914) Python 2 is going to get harder and harder to install locally so I don't think we should assume/require devs to have it installed. This PR builds on #7905 so our developer guide only has people use Python 3. --- certbot/docs/contributing.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 06d0e1b8d..c138e4f46 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -117,13 +117,11 @@ either in the same directory as ``foo.py`` or in the ``tests`` subdirectory For debugging, we recommend putting ``import ipdb; ipdb.set_trace()`` statements inside the source code. -Once you are done with your code changes, and the tests in ``foo_test.py`` pass, -run all of the unittests for Certbot with ``tox -e py27`` (this uses Python -2.7). - -Once all the unittests pass, check for sufficient test coverage using ``tox -e -py27-cover``, and then check for code style with ``tox -e lint`` (all files) or -``pylint --rcfile=.pylintrc path/to/file.py`` (single file at a time). +Once you are done with your code changes, and the tests in ``foo_test.py`` +pass, run all of the unit tests for Certbot and check for coverage with ``tox +-e py3-cover``. You should then check for code style with ``tox -e lint`` (all +files) or ``pylint --rcfile=.pylintrc path/to/file.py`` (single file at a +time). Once all of the above is successful, you may run the full test suite using ``tox --skip-missing-interpreters``. We recommend running the commands above From 751d836746ec6ca34e4c04bf8fd3cf0c67c5cef5 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 21 Apr 2020 12:47:59 -0700 Subject: [PATCH 90/92] Update using.rst to accurately describe acmev2 behavior (#7924) Fixes #7268 I removed the reference to automatically selecting which ACME protocol we use, since at some point we'll want to rip out the non-spec-compliant ACMEv1 code. --- certbot/docs/using.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 1d8d54d3a..0a0b6d1a2 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -846,17 +846,15 @@ Example usage for DNS-01 (Cloudflare API v4) (for example purposes only, do not Changing the ACME Server ======================== -By default, Certbot uses Let's Encrypt's initial production server at -https://acme-v01.api.letsencrypt.org/. You can tell Certbot to use a +By default, Certbot uses Let's Encrypt's production server at +https://acme-v02.api.letsencrypt.org/. You can tell Certbot to use a different CA by providing ``--server`` on the command line or in a :ref:`configuration file ` with the URL of the server's ACME directory. For example, if you would like to use Let's Encrypt's -new ACMEv2 server, you would add ``--server -https://acme-v02.api.letsencrypt.org/directory`` to the command line. -Certbot will automatically select which version of the ACME protocol to -use based on the contents served at the provided URL. +staging server, you would add ``--server +https://acme-staging-v02.api.letsencrypt.org/directory`` to the command line. -If you use ``--server`` to specify an ACME CA that implements a newer +If you use ``--server`` to specify an ACME CA that implements the standardized version of the spec, you may be able to obtain a certificate for a wildcard domain. Some CAs (such as Let's Encrypt) require that domain validation for wildcard domains must be done through modifications to From eed45827adde906cae06ade9742479bdf25fd8dc Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 21 Apr 2020 13:06:30 -0700 Subject: [PATCH 91/92] Remove references to the apache-parser-v2 branch (#7925) Fixes #7786. --- .azure-pipelines/main.yml | 4 ---- .travis.yml | 7 ++----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.azure-pipelines/main.yml b/.azure-pipelines/main.yml index d9609037e..eac3e451e 100644 --- a/.azure-pipelines/main.yml +++ b/.azure-pipelines/main.yml @@ -1,10 +1,6 @@ trigger: - # apache-parser-v2 is a temporary branch for doing work related to - # rewriting the parser in the Apache plugin. - - apache-parser-v2 - master pr: - - apache-parser-v2 - master - '*.x' diff --git a/.travis.yml b/.travis.yml index d3eeb1e03..f00071a6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,9 +20,6 @@ branches: # When changing these branches, please ensure the documentation under # "Running tests in CI" is still correct. only: - # apache-parser-v2 is a temporary branch for doing work related to - # rewriting the parser in the Apache plugin. - - apache-parser-v2 - master - /^\d+\.\d+\.x$/ - /^(travis-)?test-.*$/ @@ -32,9 +29,9 @@ not-on-master: ¬-on-master if: NOT (type = push AND branch = master) # Jobs for the extended test suite are executed for cron jobs and pushes to -# non-development branches. See the explanation for apache-parser-v2 above. +# non-development branches. extended-test-suite: &extended-test-suite - if: type = cron OR (type = push AND branch NOT IN (apache-parser-v2, master)) + if: type = cron OR (type = push AND branch = master) matrix: include: From 06e68cce441d30e01bf0ccd88693abd4845c113a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 21 Apr 2020 16:34:50 -0700 Subject: [PATCH 92/92] flip condition (#7927) In #7925 we accidently changed the logic here. Before it was: type = cron OR (type = push AND branch NOT IN (apache-parser-v2, master)) now it's type = cron OR (type = push AND branch = master) We want to be able to run our full test suite on things like test-* branches. The reason we had been excluding master is it has the full test suite run on it (through what Travis calls cron) nightly and not running on every commit helps prevent us on waiting on CI since our nightly tests spin up so many jobs. This PR changes things back to the intended behavior. (We could talk about changing the condition to just type = cron OR type = push if we want, but I'd rather do that in a separate PR once things like test- branches are fixed.) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f00071a6f..72cd3a408 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ not-on-master: ¬-on-master # Jobs for the extended test suite are executed for cron jobs and pushes to # non-development branches. extended-test-suite: &extended-test-suite - if: type = cron OR (type = push AND branch = master) + if: type = cron OR (type = push AND branch != master) matrix: include: