From edad9bd82b11e2e6fdc2bfd633e8c790e0da23d7 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Tue, 9 Feb 2021 04:29:31 -0600 Subject: [PATCH 01/39] Fix Sphinx manpage Building (#8646) * certbot docs: include & orphan 'man/cerbot.rst'; fixes manpage building * acme docs: include & orphan 'man/jws.rst'; fixes manpage building --- acme/docs/conf.py | 1 - acme/docs/man/jws.rst | 2 ++ certbot/docs/conf.py | 1 - certbot/docs/man/certbot.rst | 2 ++ 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/acme/docs/conf.py b/acme/docs/conf.py index 9d3c4d05c..d419326df 100644 --- a/acme/docs/conf.py +++ b/acme/docs/conf.py @@ -87,7 +87,6 @@ language = 'en' # directories to ignore when looking for source files. exclude_patterns = [ '_build', - 'man/*' ] # The reST default role (used for this markup: `text`) to use for all diff --git a/acme/docs/man/jws.rst b/acme/docs/man/jws.rst index d7ff8f405..7e83328b2 100644 --- a/acme/docs/man/jws.rst +++ b/acme/docs/man/jws.rst @@ -1 +1,3 @@ +:orphan: + .. literalinclude:: ../jws-help.txt diff --git a/certbot/docs/conf.py b/certbot/docs/conf.py index 52820b69d..254bd3edd 100644 --- a/certbot/docs/conf.py +++ b/certbot/docs/conf.py @@ -97,7 +97,6 @@ language = None # directories to ignore when looking for source files. exclude_patterns = [ '_build', - 'man', 'challenges.rst', 'ciphers.rst' ] diff --git a/certbot/docs/man/certbot.rst b/certbot/docs/man/certbot.rst index d03f3eed4..2f25504b0 100644 --- a/certbot/docs/man/certbot.rst +++ b/certbot/docs/man/certbot.rst @@ -1 +1,3 @@ +:orphan: + .. literalinclude:: ../cli-help.txt From 3d0dad8718e3b1904c1acb7da530a08053ced08f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Feb 2021 11:43:15 -0800 Subject: [PATCH 02/39] Remove dependency on six (#8650) Fixes https://github.com/certbot/certbot/issues/8494. I left the `six` dependency pinned in `tests/letstest/requirements.txt` and `tools/oldest_constraints.txt` because `six` is still a transitive dependency with our current pinnings. The extra moving around of imports is due to me using `isort` to help me keep dependencies in sorted order after replacing imports of `six`. * remove some six usage in acme * remove six from acme * remove six.add_metaclass usage * fix six.moves.zip * fix six.moves.builtins.open * six.moves server fixes * 's/six\.moves\.range/range/g' * stop using six.moves.xrange * fix urllib imports * s/six\.binary_type/bytes/g * s/six\.string_types/str/g * 's/six\.text_type/str/g' * fix six.iteritems usage * fix itervalues usage * switch from six.StringIO to io.StringIO * remove six imports * misc fixes * stop using six.reload_module * no six.PY2 * rip out six * keep six pinned in oldest constraints * fix log_test.py * update changelog --- .pylintrc | 2 +- acme/acme/challenges.py | 4 +-- acme/acme/client.py | 7 ++-- acme/acme/messages.py | 2 -- acme/acme/standalone.py | 7 ++-- acme/acme/util.py | 3 +- acme/setup.py | 1 - acme/tests/challenges_test.py | 2 +- acme/tests/client_test.py | 2 +- acme/tests/crypto_util_test.py | 12 +++---- acme/tests/standalone_test.py | 4 +-- .../certbot_apache/_internal/interfaces.py | 14 +++----- .../certbot_apache/_internal/parser.py | 3 +- certbot-apache/tests/autohsts_test.py | 1 - certbot-apache/tests/configurator_test.py | 5 ++- .../certbot_integration_tests/utils/misc.py | 4 +-- .../utils/pebble_ocsp_server.py | 2 +- .../certbot_integration_tests/utils/proxy.py | 2 +- certbot-ci/setup.py | 1 - .../certbot_compatibility_test/validator.py | 8 ++--- certbot-compatibility-test/setup.py | 1 - certbot-dns-digitalocean/setup.py | 1 - .../certbot_nginx/_internal/nginxparser.py | 13 ++++---- certbot-nginx/certbot_nginx/_internal/obj.py | 5 ++- .../certbot_nginx/_internal/parser.py | 5 ++- .../certbot_nginx/_internal/parser_obj.py | 9 +++--- certbot-nginx/tests/http_01_test.py | 3 +- certbot/CHANGELOG.md | 1 + certbot/certbot/_internal/account.py | 3 +- certbot/certbot/_internal/cli/helpful.py | 5 ++- certbot/certbot/_internal/configuration.py | 2 +- certbot/certbot/_internal/plugins/disco.py | 15 ++++----- .../certbot/_internal/plugins/selection.py | 3 +- .../certbot/_internal/plugins/standalone.py | 3 +- certbot/certbot/_internal/plugins/webroot.py | 5 ++- certbot/certbot/_internal/renewal.py | 13 ++++---- certbot/certbot/_internal/reporter.py | 2 +- certbot/certbot/_internal/storage.py | 3 +- certbot/certbot/crypto_util.py | 12 ++----- certbot/certbot/display/util.py | 4 +-- certbot/certbot/interfaces.py | 13 +++----- certbot/certbot/plugins/dns_test_common.py | 3 +- certbot/certbot/plugins/enhancements.py | 5 +-- certbot/certbot/reverter.py | 3 +- certbot/certbot/tests/acme_util.py | 3 +- certbot/certbot/tests/util.py | 28 ++++++++-------- certbot/certbot/util.py | 7 ++-- certbot/setup.py | 6 ++-- certbot/tests/account_test.py | 4 +-- certbot/tests/cli_test.py | 17 +++++----- certbot/tests/display/completer_test.py | 9 +++--- certbot/tests/display/util_test.py | 28 +++++++--------- certbot/tests/log_test.py | 24 +++++++------- certbot/tests/main_test.py | 32 ++++++++++--------- certbot/tests/plugins/disco_test.py | 5 ++- certbot/tests/plugins/manual_test.py | 3 +- certbot/tests/plugins/null_test.py | 3 +- certbot/tests/plugins/standalone_test.py | 3 +- certbot/tests/plugins/storage_test.py | 2 +- certbot/tests/plugins/webroot_test.py | 7 ++-- certbot/tests/reporter_test.py | 5 +-- certbot/tests/reverter_test.py | 3 +- certbot/tests/storage_test.py | 19 ++++++----- certbot/tests/util_test.py | 21 ++++++------ tests/letstest/multitester.py | 4 +-- 65 files changed, 196 insertions(+), 255 deletions(-) diff --git a/.pylintrc b/.pylintrc index 0e78828bd..a2468b0cf 100644 --- a/.pylintrc +++ b/.pylintrc @@ -254,7 +254,7 @@ ignore-mixin-members=yes # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis -ignored-modules=pkg_resources,confargparse,argparse,six.moves,six.moves.urllib +ignored-modules=pkg_resources,confargparse,argparse # import errors ignored only in 1.4.4 # https://bitbucket.org/logilab/pylint/commits/cd000904c9e2 diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 376e9a382..41a2aa258 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -9,7 +9,6 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose import requests -import six from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from OpenSSL import crypto @@ -145,8 +144,7 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return jobj -@six.add_metaclass(abc.ABCMeta) -class KeyAuthorizationChallenge(_TokenChallenge): +class KeyAuthorizationChallenge(_TokenChallenge, metaclass=abc.ABCMeta): """Challenge based on Key Authorization. :param response_cls: Subclass of `KeyAuthorizationChallengeResponse` diff --git a/acme/acme/client.py b/acme/acme/client.py index 6adfe4b78..c3f8c550f 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -4,6 +4,7 @@ import collections import datetime from email.utils import parsedate_tz import heapq +import http.client as http_client import logging import re import time @@ -14,8 +15,6 @@ import requests from requests.adapters import HTTPAdapter from requests.utils import parse_header_links from requests_toolbelt.adapters.source import SourceAddressAdapter -import six -from six.moves import http_client from acme import crypto_util from acme import errors @@ -248,7 +247,7 @@ class Client(ClientBase): if net is None: net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) - if isinstance(directory, six.string_types): + if isinstance(directory, str): directory = messages.Directory.from_json( net.get(directory).json()) super(Client, self).__init__(directory=directory, @@ -463,7 +462,7 @@ class Client(ClientBase): exhausted.add(authzr) if exhausted or any(authzr.body.status == messages.STATUS_INVALID - for authzr in six.itervalues(updated)): + for authzr in updated.values()): raise errors.PollError(exhausted, updated) updated_authzrs = tuple(updated[authzr] for authzr in authzrs) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 3a505843d..038cda04b 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -2,7 +2,6 @@ import json import josepy as jose -import six from acme import challenges from acme import errors @@ -68,7 +67,6 @@ def is_acme_error(err): return False -@six.python_2_unicode_compatible class Error(jose.JSONObjectWithFields, errors.Error): """ACME error. diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 7a61ba868..94397f0de 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -1,14 +1,13 @@ """Support for standalone client challenge solvers. """ import collections import functools +import http.client as http_client +import http.server as BaseHTTPServer import logging import socket +import socketserver import threading -from six.moves import BaseHTTPServer # type: ignore -from six.moves import http_client -from six.moves import socketserver # type: ignore - from acme import challenges from acme import crypto_util from acme.magic_typing import List diff --git a/acme/acme/util.py b/acme/acme/util.py index b3b0d79eb..20fa455fd 100644 --- a/acme/acme/util.py +++ b/acme/acme/util.py @@ -1,7 +1,6 @@ """ACME utilities.""" -import six def map_keys(dikt, func): """Map dictionary keys.""" - return {func(key): value for key, value in six.iteritems(dikt)} + return {func(key): value for key, value in dikt.items()} diff --git a/acme/setup.py b/acme/setup.py index 745169cbf..847ca9299 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -18,7 +18,6 @@ install_requires = [ 'requests>=2.6.0', 'requests-toolbelt>=0.3.0', 'setuptools>=39.0.1', - 'six>=1.11.0', ] dev_extras = [ diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index 22e67be3c..cc604b0de 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -1,11 +1,11 @@ """Tests for acme.challenges.""" +import urllib.parse as urllib_parse import unittest from unittest import mock import josepy as jose import OpenSSL import requests -from six.moves.urllib import parse as urllib_parse from acme import errors diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index a1be59056..6f9aecda2 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -2,6 +2,7 @@ # pylint: disable=too-many-lines import copy import datetime +import http.client as http_client import json import unittest from unittest import mock @@ -9,7 +10,6 @@ from unittest import mock import josepy as jose import OpenSSL import requests -from six.moves import http_client # pylint: disable=import-error from acme import challenges from acme import errors diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index 705a3c856..f271ad37d 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -1,14 +1,13 @@ """Tests for acme.crypto_util.""" import itertools import socket +import socketserver import threading import time import unittest import josepy as jose import OpenSSL -import six -from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import errors import test_util @@ -27,8 +26,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): class _TestServer(socketserver.TCPServer): - # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init - def server_bind(self): # pylint: disable=missing-docstring self.socket = SSLSocket(socket.socket(), certs) @@ -62,7 +59,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): self.assertRaises(errors.Error, self._probe, b'bar') def test_probe_connection_error(self): - # pylint has a hard time with six self.server.server_close() original_timeout = socket.getdefaulttimeout() try: @@ -121,9 +117,9 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): @classmethod def _get_idn_names(cls): """Returns expected names from '{cert,csr}-idnsans.pem'.""" - chars = [six.unichr(i) for i in itertools.chain(range(0x3c3, 0x400), - range(0x641, 0x6fc), - range(0x1820, 0x1877))] + chars = [chr(i) for i in itertools.chain(range(0x3c3, 0x400), + range(0x641, 0x6fc), + range(0x1820, 0x1877))] return [''.join(chars[i: i + 45]) + '.invalid' for i in range(0, len(chars), 45)] diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 5bbc2ccce..e6aa8f2d6 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -1,13 +1,13 @@ """Tests for acme.standalone.""" +import http.client as http_client import socket +import socketserver import threading import unittest from unittest import mock import josepy as jose import requests -from six.moves import http_client # pylint: disable=import-error -from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import challenges from acme import crypto_util diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 647790c41..77daa5b56 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -100,12 +100,10 @@ For this reason the internal representation of data should not ignore the case. """ import abc -import six -@six.add_metaclass(abc.ABCMeta) -class ParserNode(object): +class ParserNode(object, metaclass=abc.ABCMeta): """ ParserNode is the basic building block of the tree of such nodes, representing the structure of the configuration. It is largely meant to keep @@ -204,9 +202,7 @@ class ParserNode(object): """ -# Linter rule exclusion done because of https://github.com/PyCQA/pylint/issues/179 -@six.add_metaclass(abc.ABCMeta) # pylint: disable=abstract-method -class CommentNode(ParserNode): +class CommentNode(ParserNode, metaclass=abc.ABCMeta): """ CommentNode class is used for representation of comments within the parsed configuration structure. Because of the nature of comments, it is not able @@ -249,8 +245,7 @@ class CommentNode(ParserNode): metadata=kwargs.get('metadata', {})) # pragma: no cover -@six.add_metaclass(abc.ABCMeta) -class DirectiveNode(ParserNode): +class DirectiveNode(ParserNode, metaclass=abc.ABCMeta): """ DirectiveNode class represents a configuration directive within the configuration. It can have zero or more parameters attached to it. Because of the nature of @@ -325,8 +320,7 @@ class DirectiveNode(ParserNode): """ -@six.add_metaclass(abc.ABCMeta) -class BlockNode(DirectiveNode): +class BlockNode(DirectiveNode, metaclass=abc.ABCMeta): """ BlockNode class represents a block of nested configuration directives, comments and other blocks as its children. A BlockNode can have zero or more parameters diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 4d997545b..75be0833f 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -5,7 +5,6 @@ import logging import re import sys -import six from acme.magic_typing import Dict from acme.magic_typing import List @@ -275,7 +274,7 @@ class ApacheParser(object): while len(mods) != prev_size: prev_size = len(mods) - for match_name, match_filename in six.moves.zip( + for match_name, match_filename in zip( iterator, iterator): mod_name = self.get_arg(match_name) mod_filename = self.get_arg(match_filename) diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index d15600215..db1c46b52 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -7,7 +7,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six # pylint: disable=unused-import # six is used in mock.patch() from certbot import errors from certbot_apache._internal import constants diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 6753995c6..302bf0023 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -10,7 +10,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six # pylint: disable=unused-import # six is used in mock.patch() from acme import challenges from certbot import achallenges @@ -726,7 +725,7 @@ class MultipleVhostsTest(util.ApacheTest): # This calls open self.config.reverter.register_file_creation = mock.Mock() mock_open.side_effect = IOError - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertRaises( errors.PluginError, self.config.make_vhost_ssl, self.vh_truth[0]) @@ -1834,7 +1833,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): def test_open_module_file(self): mock_open = mock.mock_open(read_data="testing 12 3") - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertEqual(self.config._open_module_file("/nonsense/"), "testing 12 3") if __name__ == "__main__": diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index 799b079fe..e6b9f0c88 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -4,10 +4,12 @@ or outside during setup/teardown of the integration tests environment. """ import contextlib import errno +import http.server as SimpleHTTPServer import multiprocessing import os import re import shutil +import socketserver import stat import sys import tempfile @@ -23,8 +25,6 @@ from cryptography.x509 import load_pem_x509_certificate from OpenSSL import crypto import pkg_resources import requests -from six.moves import SimpleHTTPServer -from six.moves import socketserver from certbot_integration_tests.utils.constants import \ PEBBLE_ALTERNATE_ROOTS, PEBBLE_MANAGEMENT_URL diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py index b86e1cbc9..4db72998f 100755 --- a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py @@ -4,6 +4,7 @@ This runnable module interfaces itself with the Pebble management interface in o to serve a mock OCSP responder during integration tests against Pebble. """ import datetime +import http.server as BaseHTTPServer import re from cryptography import x509 @@ -13,7 +14,6 @@ from cryptography.hazmat.primitives import serialization from cryptography.x509 import ocsp from dateutil import parser import requests -from six.moves import BaseHTTPServer from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL diff --git a/certbot-ci/certbot_integration_tests/utils/proxy.py b/certbot-ci/certbot_integration_tests/utils/proxy.py index 225f98e6e..7c46e640d 100644 --- a/certbot-ci/certbot_integration_tests/utils/proxy.py +++ b/certbot-ci/certbot_integration_tests/utils/proxy.py @@ -1,12 +1,12 @@ #!/usr/bin/env python # pylint: disable=missing-module-docstring +import http.server as BaseHTTPServer import json import re import sys import requests -from six.moves import BaseHTTPServer from certbot_integration_tests.utils.misc import GracefulTCPServer diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index 3277df1c0..9f9c1f462 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -18,7 +18,6 @@ install_requires = [ 'python-dateutil', 'pyyaml', 'requests', - 'six' ] # Add pywin32 on Windows platforms to handle low-level system calls. diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index bfa03f22d..9a082523a 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -3,8 +3,6 @@ import logging import socket import requests -import six -from six.moves import xrange from acme import crypto_util from acme import errors as acme_errors @@ -19,11 +17,11 @@ class Validator(object): """Verifies the certificate presented at name is cert""" if alt_host is None: host = socket.gethostbyname(name).encode() - elif isinstance(alt_host, six.binary_type): + elif isinstance(alt_host, bytes): host = alt_host else: host = alt_host.encode() - name = name if isinstance(name, six.binary_type) else name.encode() + name = name if isinstance(name, bytes) else name.encode() try: presented_cert = crypto_util.probe_sni(name, host, port) @@ -62,7 +60,7 @@ class Validator(object): else: response = requests.get(url, allow_redirects=False) - return response.status_code in xrange(300, 309) + return response.status_code in range(300, 309) def hsts(self, name): """Test for HTTP Strict Transport Security header""" diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 0236773f0..af19b126e 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -8,7 +8,6 @@ version = '1.13.0.dev0' install_requires = [ 'certbot', 'certbot-apache', - 'six', 'requests', 'zope.interface', ] diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 8c6ac78d5..6e60444cf 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -11,7 +11,6 @@ version = '1.13.0.dev0' install_requires = [ 'python-digitalocean>=1.11', 'setuptools>=39.0.1', - 'six>=1.11.0', 'zope.interface', ] diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index 5f723dcef..f043b0e4d 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -15,7 +15,6 @@ from pyparsing import restOfLine from pyparsing import stringEnd from pyparsing import White from pyparsing import ZeroOrMore -import six from acme.magic_typing import IO, Any # pylint: disable=unused-import logger = logging.getLogger(__name__) @@ -79,7 +78,7 @@ class RawNginxDumper(object): """Iterates the dumped nginx content.""" blocks = blocks or self.blocks for b0 in blocks: - if isinstance(b0, six.string_types): + if isinstance(b0, str): yield b0 continue item = copy.deepcopy(b0) @@ -96,7 +95,7 @@ class RawNginxDumper(object): yield '}' else: # not a block - list of strings semicolon = ";" - if isinstance(item[0], six.string_types) and item[0].strip() == '#': # comment + if isinstance(item[0], str) and item[0].strip() == '#': # comment semicolon = "" yield "".join(item) + semicolon @@ -131,14 +130,14 @@ def load(_file): def dumps(blocks): - # type: (UnspacedList) -> six.text_type + # type: (UnspacedList) -> str """Dump to a Unicode string. :param UnspacedList block: The parsed tree - :rtype: six.text_type + :rtype: str """ - return six.text_type(RawNginxDumper(blocks.spaced)) + return str(RawNginxDumper(blocks.spaced)) def dump(blocks, _file): @@ -154,7 +153,7 @@ def dump(blocks, _file): _file.write(dumps(blocks)) -spacey = lambda x: (isinstance(x, six.string_types) and x.isspace()) or x == '' +spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == '' class UnspacedList(list): """Wrap a list [of lists], making any whitespace entries magically invisible""" diff --git a/certbot-nginx/certbot_nginx/_internal/obj.py b/certbot-nginx/certbot_nginx/_internal/obj.py index 4e0d8cf35..2dd02f180 100644 --- a/certbot-nginx/certbot_nginx/_internal/obj.py +++ b/certbot-nginx/certbot_nginx/_internal/obj.py @@ -1,7 +1,6 @@ """Module contains classes used by the Nginx Configurator.""" import re -import six from certbot.plugins import common @@ -211,7 +210,7 @@ class VirtualHost(object): def contains_list(self, test): """Determine if raw server block contains test list at top level """ - for i in six.moves.range(0, len(self.raw) - len(test) + 1): + for i in range(0, len(self.raw) - len(test) + 1): if self.raw[i:i + len(test)] == test: return True return False @@ -250,7 +249,7 @@ def _find_directive(directives, directive_name, match_content=None): """Find a directive of type directive_name in directives. If match_content is given, Searches for `match_content` in the directive arguments. """ - if not directives or isinstance(directives, six.string_types): + if not directives or isinstance(directives, str): return None # If match_content is None, just match on directive type. Otherwise, match on diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 72091b03f..fe5d7bb31 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -7,7 +7,6 @@ import logging import re import pyparsing -import six from acme.magic_typing import Dict from acme.magic_typing import List @@ -549,7 +548,7 @@ def _is_include_directive(entry): """ return (isinstance(entry, list) and len(entry) == 2 and entry[0] == 'include' and - isinstance(entry[1], six.string_types)) + isinstance(entry[1], str)) def _is_ssl_on_directive(entry): """Checks if an nginx parsed entry is an 'ssl on' directive. @@ -654,7 +653,7 @@ def _add_directive(block, directive, insert_at_top): directive_name = directive[0] def can_append(loc, dir_name): """ Can we append this directive to the block? """ - return loc is None or (isinstance(dir_name, six.string_types) + return loc is None or (isinstance(dir_name, str) and dir_name in REPEATABLE_DIRECTIVES) err_fmt = 'tried to insert directive "{0}" but found conflicting "{1}".' diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 390e18e4d..d616a1a99 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -4,7 +4,6 @@ raw lists of tokens from pyparsing. """ import abc import logging -import six from acme.magic_typing import List from certbot import errors @@ -152,7 +151,7 @@ class Statements(Parsable): if not isinstance(raw_list, list): raise errors.MisconfigurationError("Statements parsing expects a list!") # If there's a trailing whitespace in the list of statements, keep track of it. - if raw_list and isinstance(raw_list[-1], six.string_types) and raw_list[-1].isspace(): + if raw_list and isinstance(raw_list[-1], str) and raw_list[-1].isspace(): self._trailing_whitespace = raw_list[-1] raw_list = raw_list[:-1] self._data = [parse_raw(elem, self, add_spaces) for elem in raw_list] @@ -184,7 +183,7 @@ class Statements(Parsable): def _space_list(list_): """ Inserts whitespace between adjacent non-whitespace tokens. """ spaced_statement = [] # type: List[str] - for i in reversed(six.moves.xrange(len(list_))): + for i in reversed(range(len(list_))): spaced_statement.insert(0, list_[i]) if i > 0 and not list_[i].isspace() and not list_[i-1].isspace(): spaced_statement.insert(0, " ") @@ -206,7 +205,7 @@ class Sentence(Parsable): :returns: whether this lists is parseable by `Sentence`. """ return isinstance(lists, list) and len(lists) > 0 and \ - all(isinstance(elem, six.string_types) for elem in lists) + all(isinstance(elem, str) for elem in lists) def parse(self, raw_list, add_spaces=False): """ Parses a list of string types into this object. @@ -214,7 +213,7 @@ class Sentence(Parsable): if add_spaces: raw_list = _space_list(raw_list) if not isinstance(raw_list, list) or \ - any(not isinstance(elem, six.string_types) for elem in raw_list): + any(not isinstance(elem, str) for elem in raw_list): raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") self._data = raw_list diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index 8f0673c1f..2947b099d 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -6,7 +6,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six from acme import challenges from certbot import achallenges @@ -79,7 +78,7 @@ class HttpPerformTest(util.NginxTest): http_responses = self.http01.perform() self.assertEqual(len(http_responses), 4) - for i in six.moves.range(4): + for i in range(4): self.assertEqual(http_responses[i], acme_responses[i]) def test_mod_config(self): diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 1333d2420..370282f38 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -17,6 +17,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * The acme library no longer depends on the `security` extras from `requests` which was needed to support SNI in TLS requests when using old versions of Python 2. +* Certbot and all of its components no longer depend on the library `six`. ### Fixed diff --git a/certbot/certbot/_internal/account.py b/certbot/certbot/_internal/account.py index b4619beba..dbe111fbc 100644 --- a/certbot/certbot/_internal/account.py +++ b/certbot/certbot/_internal/account.py @@ -10,7 +10,6 @@ from cryptography.hazmat.primitives import serialization import josepy as jose import pyrfc3339 import pytz -import six from acme import fields as acme_fields from acme import messages @@ -101,7 +100,7 @@ class AccountMemoryStorage(interfaces.AccountStorage): self.accounts = initial_accounts if initial_accounts is not None else {} def find_all(self): - return list(six.itervalues(self.accounts)) + return list(self.accounts.values()) def save(self, account, client): if account.id in self.accounts: diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index 2cb506157..80afe5db4 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -7,7 +7,6 @@ import glob import sys import configargparse -import six import zope.component import zope.interface @@ -99,7 +98,7 @@ class HelpfulArgumentParser(object): if isinstance(help1, bool) and isinstance(help2, bool): self.help_arg = help1 or help2 else: - self.help_arg = help1 if isinstance(help1, six.string_types) else help2 + self.help_arg = help1 if isinstance(help1, str) else help2 short_usage = self._usage_string(plugins, self.help_arg) @@ -470,7 +469,7 @@ class HelpfulArgumentParser(object): may or may not be displayed as help topics. """ - for name, plugin_ep in six.iteritems(plugins): + for name, plugin_ep in plugins.items(): parser_or_group = self.add_group(name, description=plugin_ep.long_description) plugin_ep.plugin_cls.inject_parser_options(parser_or_group, name) diff --git a/certbot/certbot/_internal/configuration.py b/certbot/certbot/_internal/configuration.py index f1e85f9fe..1b5cf5da7 100644 --- a/certbot/certbot/_internal/configuration.py +++ b/certbot/certbot/_internal/configuration.py @@ -1,7 +1,7 @@ """Certbot user-supplied configuration.""" import copy +from urllib import parse -from six.moves.urllib import parse import zope.interface from certbot import errors diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index bed22b4ae..dbcecb067 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -5,7 +5,6 @@ import logging import sys import pkg_resources -import six import zope.interface import zope.interface.verify @@ -215,7 +214,7 @@ class PluginsRegistry(Mapping): # This prevents deadlock caused by plugins acquiring a lock # and ensures at least one concurrent Certbot instance will run # successfully. - self._plugins = collections.OrderedDict(sorted(six.iteritems(plugins))) + self._plugins = collections.OrderedDict(sorted(plugins.items())) @classmethod def find_all(cls): @@ -276,12 +275,12 @@ class PluginsRegistry(Mapping): def init(self, config): """Initialize all plugins in the registry.""" return [plugin_ep.init(config) for plugin_ep - in six.itervalues(self._plugins)] + in self._plugins.values()] def filter(self, pred): """Filter plugins based on predicate.""" return type(self)({name: plugin_ep for name, plugin_ep - in six.iteritems(self._plugins) if pred(plugin_ep)}) + in self._plugins.items() if pred(plugin_ep)}) def visible(self): """Filter plugins based on visibility.""" @@ -297,7 +296,7 @@ class PluginsRegistry(Mapping): def prepare(self): """Prepare all plugins in the registry.""" - return [plugin_ep.prepare() for plugin_ep in six.itervalues(self._plugins)] + return [plugin_ep.prepare() for plugin_ep in self._plugins.values()] def available(self): """Filter plugins based on availability.""" @@ -319,7 +318,7 @@ class PluginsRegistry(Mapping): """ # use list instead of set because PluginEntryPoint is not hashable - candidates = [plugin_ep for plugin_ep in six.itervalues(self._plugins) + candidates = [plugin_ep for plugin_ep in self._plugins.values() if plugin_ep.initialized and plugin_ep.init() is plugin] assert len(candidates) <= 1 if candidates: @@ -329,9 +328,9 @@ class PluginsRegistry(Mapping): def __repr__(self): return "{0}({1})".format( self.__class__.__name__, ','.join( - repr(p_ep) for p_ep in six.itervalues(self._plugins))) + repr(p_ep) for p_ep in self._plugins.values())) def __str__(self): if not self._plugins: return "No plugins" - return "\n\n".join(str(p_ep) for p_ep in six.itervalues(self._plugins)) + return "\n\n".join(str(p_ep) for p_ep in self._plugins.values()) diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index 0b04791c6..e5c311efe 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -3,7 +3,6 @@ from __future__ import print_function import logging -import six import zope.component from certbot import errors @@ -108,7 +107,7 @@ def pick_plugin(config, default, plugins, question, ifaces): if len(prepared) > 1: logger.debug("Multiple candidate plugins: %s", prepared) - plugin_ep = choose_plugin(list(six.itervalues(prepared)), question) + plugin_ep = choose_plugin(list(prepared.values()), question) if plugin_ep is None: return None return plugin_ep.init() diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index bbb56178c..60c9558cb 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -6,7 +6,6 @@ import socket from socket import errno as socket_errors # type: ignore import OpenSSL # pylint: disable=unused-import -import six import zope.interface from acme import challenges @@ -183,7 +182,7 @@ class Authenticator(common.Plugin): for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, servers in six.iteritems(self.servers.running()): + for port, servers in self.servers.running().items(): if not self.served[servers]: self.servers.stop(port) diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index 88e02998d..8789db604 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -4,7 +4,6 @@ import collections import json import logging -import six import zope.component import zope.interface @@ -92,7 +91,7 @@ to serve all files under specified web root ({0}).""" for achall in achalls: self.conf("map").setdefault(achall.domain, webroot_path) else: - known_webroots = list(set(six.itervalues(self.conf("map")))) + known_webroots = list(set(self.conf("map").values())) for achall in achalls: if achall.domain not in self.conf("map"): new_webroot = self._prompt_for_webroot(achall.domain, @@ -242,7 +241,7 @@ class _WebrootMapAction(argparse.Action): """Action class for parsing webroot_map.""" def __call__(self, parser, namespace, webroot_map, option_string=None): - for domains, webroot_path in six.iteritems(json.loads(webroot_map)): + for domains, webroot_path in json.loads(webroot_map).items(): webroot_path = _validate_webroot(webroot_path) namespace.webroot_map.update( (d, webroot_path) for d in cli.add_domains(namespace, domains)) diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index f4c7b4502..9fe9cb546 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -13,7 +13,6 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import ec, rsa from cryptography.hazmat.primitives.serialization import load_pem_private_key import OpenSSL -import six import zope.component from acme.magic_typing import List @@ -120,7 +119,7 @@ def _restore_webroot_config(config, renewalparams): # see https://github.com/certbot/certbot/pull/7095 if "webroot_path" in renewalparams and not cli.set_by_cli("webroot_path"): wp = renewalparams["webroot_path"] - if isinstance(wp, six.string_types): # prior to 0.1.0, webroot_path was a string + if isinstance(wp, str): # prior to 0.1.0, webroot_path was a string wp = [wp] config.webroot_path = wp @@ -155,7 +154,7 @@ def _restore_plugin_configs(config, renewalparams): for plugin_prefix in set(plugin_prefixes): plugin_prefix = plugin_prefix.replace('-', '_') - for config_item, config_value in six.iteritems(renewalparams): + for config_item, config_value in renewalparams.items(): if config_item.startswith(plugin_prefix + "_") and not cli.set_by_cli(config_item): # Values None, True, and False need to be treated specially, # As their types aren't handled correctly by configobj @@ -180,9 +179,9 @@ def restore_required_config_elements(config, renewalparams): required_items = itertools.chain( (("pref_challs", _restore_pref_challs),), - six.moves.zip(BOOL_CONFIG_ITEMS, itertools.repeat(_restore_bool)), - six.moves.zip(INT_CONFIG_ITEMS, itertools.repeat(_restore_int)), - six.moves.zip(STR_CONFIG_ITEMS, itertools.repeat(_restore_str))) + zip(BOOL_CONFIG_ITEMS, itertools.repeat(_restore_bool)), + zip(INT_CONFIG_ITEMS, itertools.repeat(_restore_int)), + zip(STR_CONFIG_ITEMS, itertools.repeat(_restore_str))) for item_name, restore_func in required_items: if item_name in renewalparams and not cli.set_by_cli(item_name): value = restore_func(item_name, renewalparams[item_name]) @@ -220,7 +219,7 @@ def _restore_pref_challs(unused_name, value): # If pref_challs has only one element, configobj saves the value # with a trailing comma so it's parsed as a list. If this comma is # removed by the user, the value is parsed as a str. - value = [value] if isinstance(value, six.string_types) else value + value = [value] if isinstance(value, str) else value return cli.parse_preferred_challenges(value) diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index d781f3d9d..64c0fbd6d 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -3,10 +3,10 @@ from __future__ import print_function import collections import logging +import queue import sys import textwrap -from six.moves import queue # type: ignore import zope.interface from certbot import interfaces diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index ff58313e5..690567a17 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -9,7 +9,6 @@ import stat import configobj import parsedatetime import pytz -import six from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.hazmat.primitives.serialization import load_pem_private_key @@ -275,7 +274,7 @@ def relevant_values(all_values): rv = dict( (option, value) - for option, value in six.iteritems(all_values) + for option, value in all_values.items() if _relevant(namespaces, option) and cli.option_was_set(option, value)) # We always save the server value to help with forward compatibility # and behavioral consistency when versions of Certbot with different diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index edd4f9eb3..f67d95d97 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -22,7 +22,6 @@ from OpenSSL import crypto from OpenSSL import SSL # type: ignore import pyrfc3339 -import six import zope.component from acme import crypto_util as acme_crypto_util @@ -215,7 +214,7 @@ def make_key(bits=1024, key_type="rsa", elliptic_curve=None): except TypeError: raise errors.Error("Unsupported elliptic curve: {}".format(elliptic_curve)) except UnsupportedAlgorithm as e: - raise six.raise_from(e, errors.Error(str(e))) + raise e from errors.Error(str(e)) _key_pem = _key.private_bytes( encoding=Encoding.PEM, format=PrivateFormat.TraditionalOpenSSL, @@ -492,14 +491,9 @@ def _notAfterBefore(cert_path, method): reformatted_timestamp = [timestamp[0:4], b"-", timestamp[4:6], b"-", timestamp[6:8], b"T", timestamp[8:10], b":", timestamp[10:12], b":", timestamp[12:]] - # pyrfc3339 always uses the type `str`. This means that in Python 2, it - # expects str/bytes and in Python 3 it expects its str type or the Python 2 - # equivalent of the type unicode. + # pyrfc3339 always uses the type `str` timestamp_bytes = b"".join(reformatted_timestamp) - if six.PY3: - timestamp_str = timestamp_bytes.decode('ascii') - else: - timestamp_str = timestamp_bytes + timestamp_str = timestamp_bytes.decode('ascii') return pyrfc3339.parse(timestamp_str) diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index c48454637..9da981892 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -71,7 +71,7 @@ def _wrap_lines(msg): def input_with_timeout(prompt=None, timeout=36000.0): """Get user input with a timeout. - Behaves the same as six.moves.input, however, an error is raised if + Behaves the same as the builtin input, however, an error is raised if a user doesn't answer after timeout seconds. The default timeout value was chosen to place it just under 12 hours for users following our advice and running Certbot twice a day. @@ -85,7 +85,7 @@ def input_with_timeout(prompt=None, timeout=36000.0): :raises errors.Error if no answer is given before the timeout """ - # use of sys.stdin and sys.stdout to mimic six.moves.input based on + # use of sys.stdin and sys.stdout to mimic the builtin input based on # https://github.com/python/cpython/blob/baf7bb30a02aabde260143136bdf5b3738a1d409/Lib/getpass.py#L129 if prompt: sys.stdout.write(prompt) diff --git a/certbot/certbot/interfaces.py b/certbot/certbot/interfaces.py index 28c6f2ac1..ddbee8ddc 100644 --- a/certbot/certbot/interfaces.py +++ b/certbot/certbot/interfaces.py @@ -1,14 +1,12 @@ """Certbot client interfaces.""" import abc -import six import zope.interface # pylint: disable=no-self-argument,no-method-argument,inherit-non-class -@six.add_metaclass(abc.ABCMeta) -class AccountStorage(object): +class AccountStorage(object, metaclass=abc.ABCMeta): """Accounts storage interface.""" @abc.abstractmethod @@ -547,8 +545,7 @@ class IReporter(zope.interface.Interface): """Prints messages to the user and clears the message queue.""" -@six.add_metaclass(abc.ABCMeta) -class RenewableCert(object): +class RenewableCert(object, metaclass=abc.ABCMeta): """Interface to a certificate lineage.""" @abc.abstractproperty @@ -613,8 +610,7 @@ class RenewableCert(object): # an update during the run or install subcommand, it should do so when # :func:`IInstaller.deploy_cert` is called. -@six.add_metaclass(abc.ABCMeta) -class GenericUpdater(object): +class GenericUpdater(object, metaclass=abc.ABCMeta): """Interface for update types not currently specified by Certbot. This class allows plugins to perform types of updates that Certbot hasn't @@ -646,8 +642,7 @@ class GenericUpdater(object): """ -@six.add_metaclass(abc.ABCMeta) -class RenewDeployer(object): +class RenewDeployer(object, metaclass=abc.ABCMeta): """Interface for update types run when a lineage is renewed This class allows plugins to perform types of updates that need to run at diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index d5044d336..a3a26a7a1 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -6,7 +6,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six from acme import challenges from certbot import achallenges @@ -31,7 +30,7 @@ class BaseAuthenticatorTest(object): challb=acme_util.DNS01, domain=DOMAIN, account_key=KEY) def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) # pylint: disable=no-member + self.assertTrue(isinstance(self.auth.more_info(), str)) # pylint: disable=no-member def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref(None), [challenges.DNS01]) # pylint: disable=no-member diff --git a/certbot/certbot/plugins/enhancements.py b/certbot/certbot/plugins/enhancements.py index 4abce2d2f..e674c32a2 100644 --- a/certbot/certbot/plugins/enhancements.py +++ b/certbot/certbot/plugins/enhancements.py @@ -1,8 +1,6 @@ """New interface style Certbot enhancements""" import abc -import six - from acme.magic_typing import Any from acme.magic_typing import Dict from acme.magic_typing import List @@ -91,8 +89,7 @@ def populate_cli(add): help=enh["cli_help"]) -@six.add_metaclass(abc.ABCMeta) -class AutoHSTSEnhancement(object): +class AutoHSTSEnhancement(object, metaclass=abc.ABCMeta): """ Enhancement interface that installer plugins can implement in order to provide functionality that configures the software to have a diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index 58e1216b7..be9d78a11 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -7,7 +7,6 @@ import sys import time import traceback -import six from certbot import errors from certbot import util @@ -518,7 +517,7 @@ class Reverter(object): # It is possible save checkpoints faster than 1 per second resulting in # collisions in the naming convention. - for _ in six.moves.range(2): + for _ in range(2): timestamp = self._checkpoint_timestamp() final_dir = os.path.join(self.config.backup_dir, timestamp) try: diff --git a/certbot/certbot/tests/acme_util.py b/certbot/certbot/tests/acme_util.py index f4a20ea86..49ca88bf5 100644 --- a/certbot/certbot/tests/acme_util.py +++ b/certbot/certbot/tests/acme_util.py @@ -2,7 +2,6 @@ import datetime import josepy as jose -import six from acme import challenges from acme import messages @@ -69,7 +68,7 @@ def gen_authzr(authz_status, domain, challs, statuses, combos=True): """ challbs = tuple( chall_to_challb(chall, status) - for chall, status in six.moves.zip(challs, statuses) + for chall, status in zip(challs, statuses) ) authz_kwargs = { "identifier": messages.Identifier( diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index acb31819f..558bfbed3 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -1,4 +1,6 @@ """Test utilities.""" +from importlib import reload as reload_module +import io import logging from multiprocessing import Event from multiprocessing import Process @@ -11,20 +13,8 @@ import warnings from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -try: - import mock - warnings.warn( - "The external mock module is being used for backwards compatibility " - "since it is available, however, future versions of Certbot's tests will " - "use unittest.mock. Be sure to update your code accordingly.", - PendingDeprecationWarning - ) -except ImportError: # pragma: no cover - from unittest import mock # type: ignore import OpenSSL import pkg_resources -import six -from six.moves import reload_module from certbot import interfaces from certbot import util @@ -36,6 +26,18 @@ from certbot.compat import filesystem from certbot.compat import os from certbot.display import util as display_util +try: + import mock + warnings.warn( + "The external mock module is being used for backwards compatibility " + "since it is available, however, future versions of Certbot's tests will " + "use unittest.mock. Be sure to update your code accordingly.", + PendingDeprecationWarning + ) +except ImportError: # pragma: no cover + from unittest import mock # type: ignore + + def vector_path(*names): """Path to a test vector.""" @@ -177,7 +179,7 @@ def patch_get_utility_with_stdout(target='zope.component.getUtility', :rtype: mock.MagicMock """ - stdout = stdout if stdout else six.StringIO() + stdout = stdout if stdout else io.StringIO() freezable_mock = _create_get_utility_mock_with_stdout(stdout) return mock.patch(target, new=freezable_mock) diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index 8db5ab34a..7bbc02f5f 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -15,7 +15,6 @@ import subprocess import sys import configargparse -import six from acme.magic_typing import Text from acme.magic_typing import Tuple @@ -153,7 +152,7 @@ def lock_dir_until_exit(dir_path): def _release_locks(): - for dir_lock in six.itervalues(_LOCKS): + for dir_lock in _LOCKS.values(): try: dir_lock.release() except: # pylint: disable=bare-except @@ -517,7 +516,7 @@ def enforce_domain_sanity(domain): """ # Unicode try: - if isinstance(domain, six.binary_type): + if isinstance(domain, bytes): domain = domain.decode('utf-8') domain.encode('ascii') except UnicodeError: @@ -579,7 +578,7 @@ def is_wildcard_domain(domain): """ wildcard_marker = b"*." # type: Union[Text, bytes] - if isinstance(domain, six.text_type): + if isinstance(domain, str): wildcard_marker = u"*." return domain.startswith(wildcard_marker) diff --git a/certbot/setup.py b/certbot/setup.py index 4ea98e574..8b2874f20 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -31,9 +31,9 @@ meta = dict(re.findall(r"""__([a-z]+)__ = '([^']+)""", read_file(init_fn))) readme = read_file(os.path.join(here, 'README.rst')) version = meta['version'] -# This package relies on PyOpenSSL, requests, and six, however, it isn't -# specified here to avoid masking the more specific request requirements in -# acme. See https://github.com/pypa/pip/issues/988 for more info. +# This package relies on PyOpenSSL and requests, however, it isn't specified +# here to avoid masking the more specific request requirements in acme. See +# https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme>=1.8.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index f9c218cd3..d0448a1db 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -276,14 +276,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() mock_open.side_effect = IOError - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertRaises( errors.AccountStorageError, self.storage.load, self.acc.id) def test_save_ioerrors(self): mock_open = mock.mock_open() mock_open.side_effect = IOError # TODO: [None, None, IOError] - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertRaises( errors.AccountStorageError, self.storage.save, self.acc, self.mock_client) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index e65ec32ec..2ec798a08 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -1,16 +1,11 @@ """Tests for certbot._internal.cli.""" import argparse import copy +from importlib import reload as reload_module +import io import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six -from six.moves import reload_module # pylint: disable=import-error - from acme import challenges from certbot import errors from certbot._internal import cli @@ -21,6 +16,12 @@ from certbot.compat import os import certbot.tests.util as test_util from certbot.tests.util import TempDirTestCase +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + PLUGINS = disco.PluginsRegistry.find_all() @@ -91,7 +92,7 @@ class ParseTest(unittest.TestCase): def _help_output(self, args): "Run a command, and return the output string for scrutiny" - output = six.StringIO() + output = io.StringIO() def write_msg(message, *args, **kwargs): # pylint: disable=missing-docstring,unused-argument output.write(message) diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 0852ab175..8ae6290bf 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -3,19 +3,20 @@ try: import readline # pylint: disable=import-error except ImportError: import certbot._internal.display.dummy_readline as readline # type: ignore +from importlib import reload as reload_module import string import sys import unittest +from certbot.compat import filesystem # pylint: disable=ungrouped-imports +from certbot.compat import os # pylint: disable=ungrouped-imports +import certbot.tests.util as test_util # pylint: disable=ungrouped-imports + try: import mock except ImportError: # pragma: no cover from unittest import mock -from six.moves import reload_module # pylint: disable=import-error -from certbot.compat import filesystem # pylint: disable=ungrouped-imports -from certbot.compat import os # pylint: disable=ungrouped-imports -import certbot.tests.util as test_util # pylint: disable=ungrouped-imports class CompleterTest(test_util.TempDirTestCase): diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 1b22d3422..30b33bbc9 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -1,20 +1,22 @@ """Test :mod:`certbot.display.util`.""" import inspect +import io import socket import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six from certbot import errors from certbot import interfaces from certbot.display import util as display_util import certbot.tests.util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + CHOICES = [("First", "Description1"), ("Second", "Description2")] TAGS = ["tag1", "tag2", "tag3"] TAGS_CHOICES = [("1", "tag1"), ("2", "tag2"), ("3", "tag3")] @@ -34,7 +36,7 @@ class InputWithTimeoutTest(unittest.TestCase): def test_input(self, prompt=None): expected = "foo bar" - stdin = six.StringIO(expected + "\n") + stdin = io.StringIO(expected + "\n") with mock.patch("certbot.compat.misc.select.select") as mock_select: mock_select.return_value = ([stdin], [], [],) self.assertEqual(self._call(prompt), expected) @@ -328,11 +330,7 @@ class FileOutputDisplayTest(unittest.TestCase): # Every IDisplay method implemented by FileDisplay must take # force_interactive to prevent workflow regressions. for name in interfaces.IDisplay.names(): - if six.PY2: - getargspec = inspect.getargspec - else: - getargspec = inspect.getfullargspec - arg_spec = getargspec(getattr(self.displayer, name)) # pylint: disable=deprecated-method + arg_spec = inspect.getfullargspec(getattr(self.displayer, name)) self.assertTrue("force_interactive" in arg_spec.args) @@ -402,12 +400,8 @@ class NoninteractiveDisplayTest(unittest.TestCase): for name in interfaces.IDisplay.names(): # pylint: disable=E1120 method = getattr(self.displayer, name) # asserts method accepts arbitrary keyword arguments - if six.PY2: - result = inspect.getargspec(method).keywords # pylint:deprecated-method - self.assertFalse(result is None) - else: - result = inspect.getfullargspec(method).varkw - self.assertFalse(result is None) + result = inspect.getfullargspec(method).varkw + self.assertFalse(result is None) class SeparateListInputTest(unittest.TestCase): diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 76764e61b..088faa0fb 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -1,15 +1,11 @@ """Tests for certbot._internal.log.""" +import io import logging import logging.handlers import sys import time import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six from acme import messages from certbot import errors @@ -19,6 +15,12 @@ from certbot.compat import filesystem from certbot.compat import os from certbot.tests import util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + class PreArgParseSetupTest(unittest.TestCase): """Tests for certbot._internal.log.pre_arg_parse_setup.""" @@ -75,7 +77,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): self.devnull = open(os.devnull, 'w') from certbot._internal.log import ColoredStreamHandler - self.stream_handler = ColoredStreamHandler(six.StringIO()) + self.stream_handler = ColoredStreamHandler(io.StringIO()) from certbot._internal.log import MemoryHandler, TempHandler self.temp_handler = TempHandler() self.temp_path = self.temp_handler.path @@ -179,7 +181,7 @@ class ColoredStreamHandlerTest(unittest.TestCase): """Tests for certbot._internal.log.ColoredStreamHandler""" def setUp(self): - self.stream = six.StringIO() + self.stream = io.StringIO() self.stream.isatty = lambda: True self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) @@ -213,7 +215,7 @@ class MemoryHandlerTest(unittest.TestCase): self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.DEBUG) self.msg = 'hi there' - self.stream = six.StringIO() + self.stream = io.StringIO() self.stream_handler = logging.StreamHandler(self.stream) from certbot._internal.log import MemoryHandler @@ -238,7 +240,7 @@ class MemoryHandlerTest(unittest.TestCase): def test_target_reset(self): self._test_log_debug() - new_stream = six.StringIO() + new_stream = io.StringIO() new_stream_handler = logging.StreamHandler(new_stream) self.handler.setTarget(new_stream_handler) self.handler.flush(force=True) @@ -325,7 +327,7 @@ class PostArgParseExceptHookTest(unittest.TestCase): def test_acme_error(self): # Get an arbitrary error code - acme_code = next(six.iterkeys(messages.ERROR_CODES)) + acme_code = next(iter(messages.ERROR_CODES)) def get_acme_error(msg): """Wraps ACME errors so the constructor takes only a msg.""" @@ -349,7 +351,7 @@ class PostArgParseExceptHookTest(unittest.TestCase): def _test_common(self, error_type, debug): """Returns the mocked logger and stderr output.""" - mock_err = six.StringIO() + mock_err = io.StringIO() def write_err(*args, **unused_kwargs): """Write error to mock_err.""" diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 5471248b4..ddd911c8d 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -4,6 +4,8 @@ from __future__ import print_function import datetime +from importlib import reload as reload_module +import io import itertools import json import shutil @@ -13,13 +15,7 @@ import traceback import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock import pytz -import six -from six.moves import reload_module # pylint: disable=import-error from certbot import crypto_util from certbot import errors @@ -39,6 +35,12 @@ from certbot.compat import os from certbot.plugins import enhancements import certbot.tests.util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.vector_path('cert_512.pem') @@ -598,7 +600,7 @@ class MainTest(test_util.ConfigTestCase): "Run the client with output streams mocked out" args = self.standard_args + args - toy_stdout = stdout if stdout else six.StringIO() + toy_stdout = stdout if stdout else io.StringIO() with mock.patch('certbot._internal.main.sys.stdout', new=toy_stdout): with mock.patch('certbot._internal.main.sys.stderr') as stderr: with mock.patch("certbot.util.atexit"): @@ -611,8 +613,8 @@ class MainTest(test_util.ConfigTestCase): self.assertEqual(1, mock_run.call_count) def test_version_string_program_name(self): - toy_out = six.StringIO() - toy_err = six.StringIO() + toy_out = io.StringIO() + toy_err = io.StringIO() with mock.patch('certbot._internal.main.sys.stdout', new=toy_out): with mock.patch('certbot._internal.main.sys.stderr', new=toy_err): try: @@ -820,7 +822,7 @@ class MainTest(test_util.ConfigTestCase): flags = ['--init', '--prepare', '--authenticators', '--installers'] for args in itertools.chain( *(itertools.combinations(flags, r) - for r in six.moves.range(len(flags)))): + for r in range(len(flags)))): self._call(['plugins'] + list(args)) @mock.patch('certbot._internal.main.plugins_disco') @@ -829,7 +831,7 @@ class MainTest(test_util.ConfigTestCase): ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() - stdout = six.StringIO() + stdout = io.StringIO() with test_util.patch_get_utility_with_stdout(stdout=stdout): _, stdout, _, _ = self._call(['plugins'], stdout) @@ -849,7 +851,7 @@ class MainTest(test_util.ConfigTestCase): _, _, _ = directory, mode, strict raise errors.Error() - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('certbot.util.set_up_core_dir') as mock_set_up_core_dir: with test_util.patch_get_utility_with_stdout(stdout=stdout): mock_set_up_core_dir.side_effect = throw_error @@ -866,7 +868,7 @@ class MainTest(test_util.ConfigTestCase): ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() - stdout = six.StringIO() + stdout = io.StringIO() with test_util.patch_get_utility_with_stdout(stdout=stdout): _, stdout, _, _ = self._call(['plugins', '--init'], stdout) @@ -884,7 +886,7 @@ class MainTest(test_util.ConfigTestCase): ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() - stdout = six.StringIO() + stdout = io.StringIO() with test_util.patch_get_utility_with_stdout(stdout=stdout): _, stdout, _, _ = self._call(['plugins', '--init', '--prepare'], stdout) @@ -1033,7 +1035,7 @@ class MainTest(test_util.ConfigTestCase): mock_certr = mock.MagicMock() mock_key = mock.MagicMock(pem='pem_key') mock_client = mock.MagicMock() - stdout = six.StringIO() + stdout = io.StringIO() mock_client.obtain_certificate.return_value = (mock_certr, 'chain', mock_key, 'csr') diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index ed13544de..a97de24b9 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -8,7 +8,6 @@ try: except ImportError: # pragma: no cover from unittest import mock import pkg_resources -import six import zope.interface from certbot import errors @@ -56,7 +55,7 @@ class PluginEntryPointTest(unittest.TestCase): EP_SA: "sa", } - for entry_point, name in six.iteritems(names): + for entry_point, name in names.items(): self.assertEqual( name, PluginEntryPoint.entry_point_to_plugin_name(entry_point, with_prefix=False)) @@ -70,7 +69,7 @@ class PluginEntryPointTest(unittest.TestCase): self.ep3: "p3:ep3", } - for entry_point, name in six.iteritems(names): + for entry_point, name in names.items(): self.assertEqual( name, PluginEntryPoint.entry_point_to_plugin_name(entry_point, with_prefix=True)) diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 7318783fd..f3c580517 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -6,7 +6,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six from acme import challenges from certbot import errors @@ -53,7 +52,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.assertRaises(errors.HookCommandNotFound, self.auth.prepare) def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) + self.assertTrue(isinstance(self.auth.more_info(), str)) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref('example.org'), diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index 47708e340..dad5b270a 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -5,7 +5,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six class InstallerTest(unittest.TestCase): @@ -16,7 +15,7 @@ class InstallerTest(unittest.TestCase): self.installer = Installer(config=mock.MagicMock(), name="null") def test_it(self): - self.assertTrue(isinstance(self.installer.more_info(), six.string_types)) + self.assertTrue(isinstance(self.installer.more_info(), str)) self.assertEqual([], self.installer.get_all_names()) self.assertEqual([], self.installer.supported_enhancements()) diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 751b9d943..596cee622 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -10,7 +10,6 @@ try: except ImportError: # pragma: no cover from unittest import mock import OpenSSL.crypto # pylint: disable=unused-import -import six from acme import challenges from acme import standalone as acme_standalone # pylint: disable=unused-import @@ -91,7 +90,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.servers = mock.MagicMock() def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) + self.assertTrue(isinstance(self.auth.more_info(), str)) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref(domain=None), diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index 4b0d1da83..2999d306e 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -33,7 +33,7 @@ class PluginStorageTest(test_util.ConfigTestCase): mock_open.side_effect = IOError self.plugin.storage.storagepath = os.path.join(self.config.config_dir, ".pluginstorage.json") - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): with mock.patch('certbot.compat.os.path.isfile', return_value=True): with mock.patch("certbot.reverter.util"): self.assertRaises(errors.PluginStorageError, diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index e57e09eae..e6fbd8e88 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -14,7 +14,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six from acme import challenges from certbot import achallenges @@ -59,7 +58,7 @@ class AuthenticatorTest(unittest.TestCase): def test_more_info(self): more_info = self.auth.more_info() - self.assertTrue(isinstance(more_info, six.string_types)) + self.assertTrue(isinstance(more_info, str)) self.assertTrue(self.path in more_info) def test_add_parser_arguments(self): @@ -83,7 +82,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertTrue(self.achall.domain in call[0][0]) self.assertTrue(all( webroot in call[0][1] - for webroot in six.itervalues(self.config.webroot_map))) + for webroot in self.config.webroot_map.values())) self.assertEqual(self.config.webroot_map[self.achall.domain], self.path) @@ -100,7 +99,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertTrue(self.achall.domain in call[0][0]) self.assertTrue(all( webroot in call[0][1] - for webroot in six.itervalues(self.config.webroot_map))) + for webroot in self.config.webroot_map.values())) @test_util.patch_get_utility() def test_new_webroot(self, mock_get_utility): diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 7d03f1821..7a37f782e 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -1,12 +1,13 @@ """Tests for certbot._internal.reporter.""" +import io import sys import unittest + try: import mock except ImportError: # pragma: no cover from unittest import mock -import six class ReporterTest(unittest.TestCase): @@ -16,7 +17,7 @@ class ReporterTest(unittest.TestCase): self.reporter = reporter.Reporter(mock.MagicMock(quiet=False)) self.old_stdout = sys.stdout # type: ignore - sys.stdout = six.StringIO() + sys.stdout = io.StringIO() def tearDown(self): sys.stdout = self.old_stdout diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index d67aa431a..af01a9a1b 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -9,7 +9,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six from certbot import errors from certbot.compat import os @@ -156,7 +155,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): act_coms = get_undo_commands(self.config.temp_checkpoint_dir) - for a_com, com in six.moves.zip(act_coms, coms): + for a_com, com in zip(act_coms, coms): self.assertEqual(a_com, com) def test_bad_register_undo_command(self): diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 914304cd4..abd496c8d 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -11,7 +11,6 @@ try: except ImportError: # pragma: no cover from unittest import mock import pytz -import six import certbot from certbot import errors @@ -286,7 +285,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version("cert"), None) def test_latest_and_next_versions(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(self.test_rc.latest_common_version(), 5) @@ -326,7 +325,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_update_link_to(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) @@ -353,12 +352,12 @@ class RenewableCertTests(BaseRenewableCertTest): os.path.basename(self.test_rc.version("cert", 8))) def test_update_all_links_to_success(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) self.assertEqual(self.test_rc.latest_common_version(), 5) - for ver in six.moves.range(1, 6): + for ver in range(1, 6): self.test_rc.update_all_links_to(ver) for kind in ALL_FOUR: self.assertEqual(ver, self.test_rc.current_version(kind)) @@ -396,11 +395,11 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version(kind), 11) def test_has_pending_deployment(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) - for ver in six.moves.range(1, 6): + for ver in range(1, 6): self.test_rc.update_all_links_to(ver) for kind in ALL_FOUR: self.assertEqual(ver, self.test_rc.current_version(kind)) @@ -498,7 +497,7 @@ class RenewableCertTests(BaseRenewableCertTest): # (to avoid instantiating parser) mock_rv.side_effect = lambda x: x - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.test_rc.update_all_links_to(3) @@ -529,7 +528,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.version("privkey", i)))) for kind in ALL_FOUR: - self.assertEqual(self.test_rc.available_versions(kind), list(six.moves.range(1, 9))) + self.assertEqual(self.test_rc.available_versions(kind), list(range(1, 9))) self.assertEqual(self.test_rc.current_version(kind), 3) # Test updating from latest version rather than old version self.test_rc.update_all_links_to(8) @@ -538,7 +537,7 @@ class RenewableCertTests(BaseRenewableCertTest): b'attempt', self.config)) for kind in ALL_FOUR: self.assertEqual(self.test_rc.available_versions(kind), - list(six.moves.range(1, 10))) + list(range(1, 10))) self.assertEqual(self.test_rc.current_version(kind), 8) with open(self.test_rc.version("fullchain", 9)) as f: self.assertEqual(f.read(), "last" + "attempt") diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 7b510fbb6..18947c342 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -1,21 +1,22 @@ """Tests for certbot.util.""" import argparse import errno +from importlib import reload as reload_module +import io import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six -from six.moves import reload_module # pylint: disable=import-error - from certbot import errors from certbot.compat import filesystem from certbot.compat import os import certbot.tests.util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + class EnvNoSnapForExternalCallsTest(unittest.TestCase): """Tests for certbot.util.env_no_snap_for_external_calls.""" @@ -265,11 +266,11 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): def test_multiple(self): items = [] - for _ in six.moves.range(10): + for _ in range(10): items.append(self._call("wow")) f, name = items[-1] self.assertTrue(isinstance(f, file_type)) - self.assertTrue(isinstance(name, six.string_types)) + self.assertTrue(isinstance(name, str)) self.assertTrue("wow-0009.conf" in name) for f, _ in items: f.close() @@ -361,7 +362,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): def test_help(self): self._call("--old-option", 2) - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch("sys.stdout", new=stdout): try: self.parser.parse_args(["-h"]) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 5ad1d8c15..83a92e6dc 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -39,11 +39,11 @@ import socket import sys import time import traceback +import urllib.error as urllib_error +import urllib.request as urllib_request import boto3 from botocore.exceptions import ClientError -from six.moves.urllib import error as urllib_error -from six.moves.urllib import request as urllib_request import yaml from fabric import Config From cf062f4c6dd7642cf124f5e7279fdc3f6ded063b Mon Sep 17 00:00:00 2001 From: Steffen Neumann Date: Tue, 9 Feb 2021 21:18:29 +0100 Subject: [PATCH 03/39] Fix ubuntu package name (#8654) Since Ubuntu 18.04 there is python3-certbot-apache which should be the recommended version. The Debian package names should probably be updated accordingly. --- certbot/docs/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index 4366080e0..8ae1c82f2 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -191,7 +191,7 @@ Optionally to install the Certbot Apache plugin, you can use: .. code-block:: shell - sudo apt-get install python-certbot-apache + sudo apt-get install python3-certbot-apache **Fedora** From c59775c3c069ce7106eca63e8f16294d99fbb021 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 11 Feb 2021 00:17:51 +0100 Subject: [PATCH 04/39] Disable certbot-auto upgrade on RHEL-like systems (#8653) Fixes #8637 * Disable upgrade for RHEL-like systems * Remove letstest on Amazon Linux * Update changelog --- certbot/CHANGELOG.md | 1 + letsencrypt-auto-source/letsencrypt-auto | 1 + letsencrypt-auto-source/letsencrypt-auto.template | 1 + tests/letstest/auto_targets.yaml | 7 ------- tests/letstest/scripts/test_leauto_upgrades.sh | 2 +- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 370282f38..eac783bdb 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -18,6 +18,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). which was needed to support SNI in TLS requests when using old versions of Python 2. * Certbot and all of its components no longer depend on the library `six`. +* The update of certbot-auto itself is now disabled on all RHEL-like systems. ### Fixed diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 14e71c615..224abaf01 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -806,6 +806,7 @@ elif [ -f /etc/mageia-release ]; then NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 783268571..70b75176e 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -328,6 +328,7 @@ elif [ -f /etc/mageia-release ]; then NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. diff --git a/tests/letstest/auto_targets.yaml b/tests/letstest/auto_targets.yaml index 164580e86..01d410227 100644 --- a/tests/letstest/auto_targets.yaml +++ b/tests/letstest/auto_targets.yaml @@ -57,10 +57,3 @@ targets: type: centos virt: hvm user: centos - #----------------------------------------------------------------------------- - # Amazon Linux - - ami: ami-0ff8a91507f77f867 - name: amazon - type: centos - virt: hvm - user: ec2-user diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index d0b941736..407a865f2 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -156,7 +156,7 @@ fi # Finally, we check if our local server received more requests. Over time, # we'll move more and more OSes into this case until it this is the expected # behavior on all systems. -if [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue; then +if [ -f /etc/redhat-release ]; then if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then echo our local server received unexpected requests exit 1 From c0eccdd35845161ced2ba56f77131c05ae2634fd Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 13 Feb 2021 01:14:46 +0100 Subject: [PATCH 05/39] Deprecate certbot-auto specific flags (#8641) This PR deprecates the certbot-auto specific CLI flags, in the perspective of removing them in a future release as said in #8483. * Deprecate certbot-auto specific flags * Update changelog * Clean tests Co-authored-by: Brad Warren --- certbot/CHANGELOG.md | 2 ++ certbot/certbot/_internal/cli/__init__.py | 27 +++++-------------- .../certbot/_internal/cli/cli_constants.py | 8 +++++- certbot/tests/cli_test.py | 4 --- 4 files changed, 15 insertions(+), 26 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index eac783bdb..1067c3b92 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -10,6 +10,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed +* CLI flags `--os-packages-only`, `--no-self-upgrade`, `--no-bootstrap` and `--no-permissions-check`, + which are related to certbot-auto, are deprecated and will be removed in a future release. * Certbot no longer conditionally depends on an external mock module. Certbot's test API will continue to use it if it is available for backwards compatibility, however, this behavior has been deprecated and will be removed diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index c69bb3564..8d2f7c329 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -249,27 +249,6 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): default=flag_default("duplicate"), help="Allow making a certificate lineage that duplicates an existing one " "(both can be renewed in parallel)") - helpful.add( - "automation", "--os-packages-only", action="store_true", - default=flag_default("os_packages_only"), - help="(certbot-auto only) install OS package dependencies and then stop") - helpful.add( - "automation", "--no-self-upgrade", action="store_true", - default=flag_default("no_self_upgrade"), - help="(certbot-auto only) prevent the certbot-auto script from" - " upgrading itself to newer released versions (default: Upgrade" - " automatically)") - helpful.add( - "automation", "--no-bootstrap", action="store_true", - default=flag_default("no_bootstrap"), - help="(certbot-auto only) prevent the certbot-auto script from" - " installing OS-level dependencies (default: Prompt to install " - " OS-wide dependencies, but exit if the user says 'No')") - helpful.add( - "automation", "--no-permissions-check", action="store_true", - default=flag_default("no_permissions_check"), - help="(certbot-auto only) skip the check on the file system" - " permissions of the certbot-auto script") helpful.add( ["automation", "renew", "certonly", "run"], "-q", "--quiet", dest="quiet", action="store_true", @@ -451,6 +430,12 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): default=flag_default("autorenew"), dest="autorenew", help="Disable auto renewal of certificates.") + # Deprecated arguments + helpful.add_deprecated_argument("--os-packages-only", 0) + helpful.add_deprecated_argument("--no-self-upgrade", 0) + helpful.add_deprecated_argument("--no-bootstrap", 0) + helpful.add_deprecated_argument("--no-permissions-check", 0) + # Populate the command line parameters for new style enhancements enhancements.populate_cli(helpful.add) diff --git a/certbot/certbot/_internal/cli/cli_constants.py b/certbot/certbot/_internal/cli/cli_constants.py index dc199e152..bd25f9bee 100644 --- a/certbot/certbot/_internal/cli/cli_constants.py +++ b/certbot/certbot/_internal/cli/cli_constants.py @@ -109,4 +109,10 @@ VAR_MODIFIERS = {"account": {"server",}, # This is a list of all CLI options that we have ever deprecated. It lets us # opt out of the default detection, which can interact strangely with option # deprecation. See https://github.com/certbot/certbot/issues/8540 for more info. -DEPRECATED_OPTIONS = {"manual_public_ip_logging_ok",} +DEPRECATED_OPTIONS = { + "manual_public_ip_logging_ok", + "os_packages_only", + "no_self_upgrade", + "no_bootstrap", + "no_permissions_check", +} diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 2ec798a08..fca2b3e3e 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -480,10 +480,6 @@ class ParseTest(unittest.TestCase): for topic in ['all', 'plugins', 'dns-route53']: self.assertFalse('certbot-route53:auth' in self._help_output([help_flag, topic])) - def test_no_permissions_check_accepted(self): - namespace = self.parse(["--no-permissions-check"]) - self.assertTrue(namespace.no_permissions_check) - class DefaultTest(unittest.TestCase): """Tests for certbot._internal.cli._Default.""" From ef265eccaf125a51a8c260d28e2f501e1c3111dc Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Sun, 21 Feb 2021 23:23:42 +0100 Subject: [PATCH 06/39] Remove import fallback for collections.abc (#8674) --- acme/acme/messages.py | 8 +------- certbot/certbot/_internal/plugins/disco.py | 7 +------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 038cda04b..44ecb143c 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,5 +1,6 @@ """ACME protocol messages.""" import json +from collections.abc import Hashable import josepy as jose @@ -10,13 +11,6 @@ from acme import jws from acme import util from acme.mixins import ResourceMixin -try: - from collections.abc import Hashable -except ImportError: # pragma: no cover - from collections import Hashable - - - OLD_ERROR_PREFIX = "urn:acme:error:" ERROR_PREFIX = "urn:ietf:params:acme:error:" diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index dbcecb067..d19fdd3ef 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -3,6 +3,7 @@ import collections import itertools import logging import sys +from collections.abc import Mapping import pkg_resources import zope.interface @@ -14,12 +15,6 @@ from certbot import interfaces from certbot._internal import constants from certbot.compat import os -try: - # Python 3.3+ - from collections.abc import Mapping -except ImportError: # pragma: no cover - from collections import Mapping - logger = logging.getLogger(__name__) PREFIX_FREE_DISTRIBUTIONS = [ From 0f3f07b5cb3bf1d2c3d8834a83022fd11d7566fc Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Sun, 21 Feb 2021 23:34:56 +0100 Subject: [PATCH 07/39] Removed backport of unittest.assertLogs (#8673) * Removed backport of unittest.assertLogs * Update parser_test.py --- certbot-nginx/tests/test_log_util.py | 125 --------------------------- certbot-nginx/tests/test_util.py | 3 +- 2 files changed, 1 insertion(+), 127 deletions(-) delete mode 100644 certbot-nginx/tests/test_log_util.py diff --git a/certbot-nginx/tests/test_log_util.py b/certbot-nginx/tests/test_log_util.py deleted file mode 100644 index 7aebf2151..000000000 --- a/certbot-nginx/tests/test_log_util.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Backport for `TestCase.assertLogs()`. - -Most of the idea and code are from CPython implementation. -https://github.com/python/cpython/blob/b76518d43fb82ed9e5d27025d18c90a23d525c90/Lib/unittest/case.py -""" -import logging -import collections - -__all__ = ['AssertLogsMixin'] - -LoggingWatcher = collections.namedtuple('LoggingWatcher', ['records', 'output']) - - -class CapturingHandler(logging.Handler): - """ - A logging handler capturing all (raw and formatted) logging output. - """ - - def __init__(self): - super(CapturingHandler, self).__init__() - self.watcher = LoggingWatcher([], []) - - def flush(self): - pass - - def emit(self, record): - self.watcher.records.append(record) - self.watcher.output.append(self.format(record)) - - - -class AssertLogsContext(object): - """ - A context manager used to implement `TestCase.assertLogs()`. - """ - - LOGGING_FORMAT = '%(levelname)s:%(name)s:%(message)s' - - def __init__(self, test_case, logger_name, level): - self.test_case = test_case - - self.logger_name = logger_name - self.logger_states = None - self.logger = None - - if level: - # pylint: disable=protected-access,no-member - try: - # In Python 3.x - name_to_level = logging._nameToLevel # type: ignore - except AttributeError: - # In Python 2.7 - name_to_level = logging._levelNames # type: ignore - - self.level = name_to_level.get(level, level) - else: - self.level = logging.INFO - - self.watcher = None - - def _save_logger_states(self): - self.logger_states = (self.logger.handlers[:], self.logger.level, self.logger.propagate) - - def _restore_logger_states(self): - self.logger.handlers, self.logger.level, self.logger.propagate = self.logger_states - - def __enter__(self): - if isinstance(self.logger_name, logging.Logger): - self.logger = self.logger_name - else: - self.logger = logging.getLogger(self.logger_name) - - formatter = logging.Formatter(self.LOGGING_FORMAT) - - handler = CapturingHandler() - handler.setFormatter(formatter) - - self._save_logger_states() - self.logger.handlers = [handler] - self.logger.setLevel(self.level) - self.logger.propagate = False - - self.watcher = handler.watcher - return handler.watcher - - def __exit__(self, exc_type, exc_value, tb): - self._restore_logger_states() - - if exc_type is not None: - # let unexpected exceptions pass through - return - - if not self.watcher.records: - self._raiseFailure( - "no logs of level {} or higher triggered on {}" - .format(logging.getLevelName(self.level), self.logger.name)) - - def _raiseFailure(self, message): - message = self.test_case._formatMessage(None, message) # pylint: disable=protected-access - raise self.test_case.failureException(message) - - -class AssertLogsMixin(object): - """ - A mixin that implements `TestCase.assertLogs()`. - """ - - def assertLogs(self, logger=None, level=None): - """Fail unless a log message of level *level* or higher is emitted - on *logger_name* or its children. If omitted, *level* defaults to - INFO and *logger* defaults to the root logger. - This method must be used as a context manager, and will yield - a recording object with two attributes: `output` and `records`. - At the end of the context manager, the `output` attribute will - be a list of the matching formatted log messages and the - `records` attribute will be a list of the corresponding LogRecord - objects. - Example:: - with self.assertLogs('foo', level='INFO') as cm: - logging.getLogger('foo').info('first message') - logging.getLogger('foo.bar').error('second message') - self.assertEqual(cm.output, ['INFO:foo:first message', - 'ERROR:foo.bar:second message']) - """ - return AssertLogsContext(self, logger, level) diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index f545dc5bc..ee53e8e1e 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -17,10 +17,9 @@ from certbot.plugins import common from certbot.tests import util as test_util from certbot_nginx._internal import configurator from certbot_nginx._internal import nginxparser -import test_log_util -class NginxTest(test_log_util.AssertLogsMixin, test_util.ConfigTestCase): +class NginxTest(test_util.ConfigTestCase): def setUp(self): super(NginxTest, self).setUp() From c43f4fe518b48465b5cb886f2b1df1f3aed17616 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 23 Feb 2021 13:20:04 -0800 Subject: [PATCH 08/39] upgrade to 3.8.8 (#8682) Fixes https://github.com/certbot/certbot/issues/8681. https://python-security.readthedocs.io/vuln/ctypes-buffer-overflow-pycarg_repr.html is the best resource I found linking to the original Python bug, when each Python branch was fixed, etc. --- windows-installer/construct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows-installer/construct.py b/windows-installer/construct.py index 0684b3c25..60834e7e5 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -9,7 +9,7 @@ import sys import tempfile import time -PYTHON_VERSION = (3, 8, 6) +PYTHON_VERSION = (3, 8, 8) PYTHON_BITNESS = 32 PYWIN32_VERSION = 300 # do not forget to edit pywin32 dependency accordingly in setup.py NSIS_VERSION = '3.06.1' From c3d6fca3eb1f24616bb784865662adfa048f9bf9 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 24 Feb 2021 00:29:52 +0100 Subject: [PATCH 09/39] Make certbot constraint file independent from certbot-auto + update cryptography (#8649) * Refactor to not depend on certbot-auto dependencies pinning anymore * Update constraints * Replaces references * Upgrade AWS dependencies pinning * Fix script * Fix Windows installer builds * Fixing sdists letstest script * Pin cryptography on 3.1.1 specifically for RHEL/CentOS 7 to avoid build failures during test_sdists test. * Finish fix * Fix VERSION_ID in RHEL 7 --- snap/snapcraft.yaml | 6 +- tests/letstest/scripts/test_sdists.sh | 22 +- tools/certbot_constraints.txt | 262 ++++++++++++++++++ tools/dev_constraints.txt | 6 +- tools/docker/core/Dockerfile | 4 - tools/pip_install.py | 6 +- .../rebuild_certbot_constraints.py | 52 ++-- tools/snap/generate_dnsplugins_all.sh | 2 +- windows-installer/construct.py | 18 +- 9 files changed, 325 insertions(+), 53 deletions(-) create mode 100644 tools/certbot_constraints.txt rename letsencrypt-auto-source/rebuild_dependencies.py => tools/rebuild_certbot_constraints.py (84%) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index c9061ecb3..d53fba88b 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -73,10 +73,10 @@ parts: build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev] build-environment: - SNAPCRAFT_PYTHON_VENV_ARGS: --upgrade - # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the + # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the # parts.[part_name].constraints option available in snapcraft.yaml when the Python plugin is # used. This is done to let these constraints be applied not only on the certbot package - # build, but also on any isolated build that pip could trigger when building wheels for + # build, but also on any isolated build that pip could trigger when building wheels for # dependencies. See https://github.com/certbot/certbot/pull/8443 for more info. - PIP_CONSTRAINT: $SNAPCRAFT_PART_SRC/snap-constraints.txt override-build: | @@ -85,7 +85,7 @@ parts: snapcraftctl build override-pull: | snapcraftctl pull - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/letsencrypt-auto-source/pieces/dependency-requirements.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_constraints.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index aa12d5610..becdd6d9a 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -12,13 +12,21 @@ sudo $BOOTSTRAP_SCRIPT # We strip the hashes because the venv creation script includes unhashed # constraints in the commands given to pip and the mix of hashed and unhashed # packages makes pip error out. -python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > requirements.txt -# We also strip out the requirement for enum34 because it cannot be installed -# in newer versions of Python 3, tools/strip_hashes.py removes the environment -# marker that'd normally prevent it from being installed, and this package is -# not needed for any OS tested here. -sed -i '/enum34/d' requirements.txt -CERTBOT_PIP_NO_BINARY=:all: tools/venv.py --requirement requirements.txt +python3 tools/strip_hashes.py tools/pipstrap_constraints.txt > constraints.txt +python3 tools/strip_hashes.py tools/certbot_constraints.txt > requirements.txt + +# We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7 +# because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been +# dropped on cryptography>=3.2 and pyOpenSSL>=20.0.0. +# Using this old version of OpenSSL would break the cryptography and pyOpenSSL wheels builds. +if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then + sed -i 's|cryptography==.*|cryptography==3.1.1|g' requirements.txt + sed -i 's|pyOpenSSL==.*|pyOpenSSL==19.1.0|g' requirements.txt +fi + +python3 -m venv $VENV_PATH +$VENV_PATH/bin/python3 tools/pipstrap.py +PIP_CONSTRAINT=constraints.txt PIP_NO_BINARY=:all: $VENV_PATH/bin/python3 -m pip install --requirement requirements.txt . "$VENV_PATH/bin/activate" # pytest is needed to run tests on some of our packages so we install a pinned version here. tools/pip_install.py pytest diff --git a/tools/certbot_constraints.txt b/tools/certbot_constraints.txt new file mode 100644 index 000000000..77bfef9db --- /dev/null +++ b/tools/certbot_constraints.txt @@ -0,0 +1,262 @@ +# This is the flattened list of pinned packages to build certbot deployable artifacts. +# To generate this, do (with docker and package hashin installed): +# ``` +# tools/rebuild_certbot_contraints.py \ +# tools/certbot_constraints.txt +# ``` +# If you want to update a single dependency, run commands similar to these: +# ``` +# pip install hashin +# hashin -r dependency-requirements.txt cryptography==1.5.2 +# ``` +ConfigArgParse==1.2.3 \ + --hash=sha256:edd17be986d5c1ba2e307150b8e5f5107aba125f3574dddd02c85d5cdcfd37dc +certifi==2020.12.5 \ + --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ + --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 +cffi==1.14.4 \ + --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \ + --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \ + --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \ + --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \ + --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \ + --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \ + --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \ + --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \ + --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \ + --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \ + --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \ + --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \ + --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \ + --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \ + --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \ + --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \ + --hash=sha256:7ef7d4ced6b325e92eb4d3502946c78c5367bc416398d387b39591532536734e \ + --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \ + --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \ + --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \ + --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \ + --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \ + --hash=sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01 \ + --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \ + --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \ + --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \ + --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \ + --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \ + --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \ + --hash=sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e \ + --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \ + --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \ + --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \ + --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \ + --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \ + --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \ + --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f +chardet==4.0.0 \ + --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ + --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 +configobj==5.0.6 \ + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 +cryptography==3.3.2 \ + --hash=sha256:0d7b69674b738068fa6ffade5c962ecd14969690585aaca0a1b1fc9058938a72 \ + --hash=sha256:1bd0ccb0a1ed775cd7e2144fe46df9dc03eefd722bbcf587b3e0616ea4a81eff \ + --hash=sha256:3c284fc1e504e88e51c428db9c9274f2da9f73fdf5d7e13a36b8ecb039af6e6c \ + --hash=sha256:49570438e60f19243e7e0d504527dd5fe9b4b967b5a1ff21cc12b57602dd85d3 \ + --hash=sha256:541dd758ad49b45920dda3b5b48c968f8b2533d8981bcdb43002798d8f7a89ed \ + --hash=sha256:5a60d3780149e13b7a6ff7ad6526b38846354d11a15e21068e57073e29e19bed \ + --hash=sha256:7951a966613c4211b6612b0352f5bf29989955ee592c4a885d8c7d0f830d0433 \ + --hash=sha256:922f9602d67c15ade470c11d616f2b2364950602e370c76f0c94c94ae672742e \ + --hash=sha256:a0f0b96c572fc9f25c3f4ddbf4688b9b38c69836713fb255f4a2715d93cbaf44 \ + --hash=sha256:a777c096a49d80f9d2979695b835b0f9c9edab73b59e4ceb51f19724dda887ed \ + --hash=sha256:a9a4ac9648d39ce71c2f63fe7dc6db144b9fa567ddfc48b9fde1b54483d26042 \ + --hash=sha256:aa4969f24d536ae2268c902b2c3d62ab464b5a66bcb247630d208a79a8098e9b \ + --hash=sha256:c7390f9b2119b2b43160abb34f63277a638504ef8df99f11cb52c1fda66a2e6f \ + --hash=sha256:e18e6ab84dfb0ab997faf8cca25a86ff15dfea4027b986322026cc99e0a892da +distro==1.5.0 \ + --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ + --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 +idna==2.10 \ + --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ + --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 +josepy==1.6.0 \ + --hash=sha256:0aab1c3ceffe045e7fd5bcfe7685e27e9d2758518d9ba7116b5de34087e70bf5 \ + --hash=sha256:65f077fc5902aca1e140ddb000e7abb081d5fb8421db60b6071076ef81c5bd27 +parsedatetime==2.6 \ + --hash=sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455 \ + --hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b +pyOpenSSL==20.0.1 \ + --hash=sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51 \ + --hash=sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b +pyRFC3339==1.1 \ + --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ + --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a +pycparser==2.20 \ + --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ + --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 +pyparsing==2.4.7 \ + --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ + --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b +python-augeas==0.5.0 \ + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 +pytz==2021.1 \ + --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \ + --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 +requests==2.25.1 \ + --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ + --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e +requests-toolbelt==0.9.1 \ + --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ + --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced +urllib3==1.26.3 \ + --hash=sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80 \ + --hash=sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73 +zope.component==4.6.2 \ + --hash=sha256:607628e4c84f7887a69a958542b5c304663e726b73aba0882e3a3f059bff14f3 \ + --hash=sha256:91628918218b3e6f6323de2a7b845e09ddc5cae131c034896c051b084bba3c92 +zope.deferredimport==4.3.1 \ + --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \ + --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a +zope.deprecation==4.4.0 \ + --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ + --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 +zope.event==4.5.0 \ + --hash=sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42 \ + --hash=sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330 +zope.hookable==5.0.1 \ + --hash=sha256:0194b9b9e7f614abba60c90b231908861036578297515d3d6508eb10190f266d \ + --hash=sha256:0c2977473918bdefc6fa8dfb311f154e7f13c6133957fe649704deca79b92093 \ + --hash=sha256:17b8bdb3b77e03a152ca0d5ca185a7ae0156f5e5a2dbddf538676633a1f7380f \ + --hash=sha256:29d07681a78042cdd15b268ae9decffed9ace68a53eebeb61d65ae931d158841 \ + --hash=sha256:36fb1b35d1150267cb0543a1ddd950c0bc2c75ed0e6e92e3aaa6ac2e29416cb7 \ + --hash=sha256:3aed60c2bb5e812bbf9295c70f25b17ac37c233f30447a96c67913ba5073642f \ + --hash=sha256:3cac1565cc768911e72ca9ec4ddf5c5109e1fef0104f19f06649cf1874943b60 \ + --hash=sha256:3d4bc0cc4a37c3cd3081063142eeb2125511db3c13f6dc932d899c512690378e \ + --hash=sha256:3f73096f27b8c28be53ffb6604f7b570fbbb82f273c6febe5f58119009b59898 \ + --hash=sha256:522d1153d93f2d48aa0bd9fb778d8d4500be2e4dcf86c3150768f0e3adbbc4ef \ + --hash=sha256:523d2928fb7377bbdbc9af9c0b14ad73e6eaf226349f105733bdae27efd15b5a \ + --hash=sha256:5848309d4fc5c02150a45e8f8d2227e5bfda386a508bbd3160fed7c633c5a2fa \ + --hash=sha256:6781f86e6d54a110980a76e761eb54590630fd2af2a17d7edf02a079d2646c1d \ + --hash=sha256:6fd27921ebf3aaa945fa25d790f1f2046204f24dba4946f82f5f0a442577c3e9 \ + --hash=sha256:70d581862863f6bf9e175e85c9d70c2d7155f53fb04dcdb2f73cf288ca559a53 \ + --hash=sha256:81867c23b0dc66c8366f351d00923f2bc5902820a24c2534dfd7bf01a5879963 \ + --hash=sha256:81db29edadcbb740cd2716c95a297893a546ed89db1bfe9110168732d7f0afdd \ + --hash=sha256:86bd12624068cea60860a0759af5e2c3adc89c12aef6f71cf12f577e28deefe3 \ + --hash=sha256:9c184d8f9f7a76e1ced99855ccf390ffdd0ec3765e5cbf7b9cada600accc0a1e \ + --hash=sha256:acc789e8c29c13555e43fe4bf9fcd15a65512c9645e97bbaa5602e3201252b02 \ + --hash=sha256:afaa740206b7660d4cc3b8f120426c85761f51379af7a5b05451f624ad12b0af \ + --hash=sha256:b5f5fa323f878bb16eae68ea1ba7f6c0419d4695d0248bed4b18f51d7ce5ab85 \ + --hash=sha256:bd89e0e2c67bf4ac3aca2a19702b1a37269fb1923827f68324ac2e7afd6e3406 \ + --hash=sha256:c212de743283ec0735db24ec6ad913758df3af1b7217550ff270038062afd6ae \ + --hash=sha256:ca553f524293a0bdea05e7f44c3e685e4b7b022cb37d87bc4a3efa0f86587a8d \ + --hash=sha256:cab67065a3db92f636128d3157cc5424a145f82d96fb47159c539132833a6d36 \ + --hash=sha256:d3b3b3eedfdbf6b02898216e85aa6baf50207f4378a2a6803d6d47650cd37031 \ + --hash=sha256:d9f4a5a72f40256b686d31c5c0b1fde503172307beb12c1568296e76118e402c \ + --hash=sha256:df5067d87aaa111ed5d050e1ee853ba284969497f91806efd42425f5348f1c06 \ + --hash=sha256:e2587644812c6138f05b8a41594a8337c6790e3baf9a01915e52438c13fc6bef \ + --hash=sha256:e27fd877662db94f897f3fd532ef211ca4901eb1a70ba456f15c0866a985464a \ + --hash=sha256:e427ebbdd223c72e06ba94c004bb04e996c84dec8a0fa84e837556ae145c439e \ + --hash=sha256:e583ad4309c203ef75a09d43434cf9c2b4fa247997ecb0dcad769982c39411c7 \ + --hash=sha256:e760b2bc8ece9200804f0c2b64d10147ecaf18455a2a90827fbec4c9d84f3ad5 \ + --hash=sha256:ea9a9cc8bcc70e18023f30fa2f53d11ae069572a162791224e60cd65df55fb69 \ + --hash=sha256:ecb3f17dce4803c1099bd21742cd126b59817a4e76a6544d31d2cca6e30dbffd \ + --hash=sha256:ed794e3b3de42486d30444fb60b5561e724ee8a2d1b17b0c2e0f81e3ddaf7a87 \ + --hash=sha256:ee885d347279e38226d0a437b6a932f207f691c502ee565aba27a7022f1285df \ + --hash=sha256:fd5e7bc5f24f7e3d490698f7b854659a9851da2187414617cd5ed360af7efd63 \ + --hash=sha256:fe45f6870f7588ac7b2763ff1ce98cce59369717afe70cc353ec5218bc854bcc +zope.interface==5.2.0 \ + --hash=sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1 \ + --hash=sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d \ + --hash=sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123 \ + --hash=sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232 \ + --hash=sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549 \ + --hash=sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102 \ + --hash=sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5 \ + --hash=sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45 \ + --hash=sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00 \ + --hash=sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc \ + --hash=sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7 \ + --hash=sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104 \ + --hash=sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034 \ + --hash=sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3 \ + --hash=sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3 \ + --hash=sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4 \ + --hash=sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86 \ + --hash=sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96 \ + --hash=sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546 \ + --hash=sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb \ + --hash=sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3 \ + --hash=sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b \ + --hash=sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b \ + --hash=sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec \ + --hash=sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae \ + --hash=sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e \ + --hash=sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386 \ + --hash=sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2 \ + --hash=sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a \ + --hash=sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d \ + --hash=sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a \ + --hash=sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24 \ + --hash=sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d \ + --hash=sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b \ + --hash=sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50 \ + --hash=sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523 \ + --hash=sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a \ + --hash=sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095 \ + --hash=sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a \ + --hash=sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520 \ + --hash=sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65 \ + --hash=sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11 \ + --hash=sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c \ + --hash=sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7 \ + --hash=sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332 \ + --hash=sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e \ + --hash=sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c \ + --hash=sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7 \ + --hash=sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20 \ + --hash=sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc \ + --hash=sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd \ + --hash=sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537 +zope.proxy==4.3.5 \ + --hash=sha256:00573dfa755d0703ab84bb23cb6ecf97bb683c34b340d4df76651f97b0bab068 \ + --hash=sha256:092049280f2848d2ba1b57b71fe04881762a220a97b65288bcb0968bb199ec30 \ + --hash=sha256:0cbd27b4d3718b5ec74fc65ffa53c78d34c65c6fd9411b8352d2a4f855220cf1 \ + --hash=sha256:17fc7e16d0c81f833a138818a30f366696653d521febc8e892858041c4d88785 \ + --hash=sha256:19577dfeb70e8a67249ba92c8ad20589a1a2d86a8d693647fa8385408a4c17b0 \ + --hash=sha256:207aa914576b1181597a1516e1b90599dc690c095343ae281b0772e44945e6a4 \ + --hash=sha256:219a7db5ed53e523eb4a4769f13105118b6d5b04ed169a283c9775af221e231f \ + --hash=sha256:2b50ea79849e46b5f4f2b0247a3687505d32d161eeb16a75f6f7e6cd81936e43 \ + --hash=sha256:5903d38362b6c716e66bbe470f190579c530a5baf03dbc8500e5c2357aa569a5 \ + --hash=sha256:5c24903675e271bd688c6e9e7df5775ac6b168feb87dbe0e4bcc90805f21b28f \ + --hash=sha256:5ef6bc5ed98139e084f4e91100f2b098a0cd3493d4e76f9d6b3f7b95d7ad0f06 \ + --hash=sha256:61b55ae3c23a126a788b33ffb18f37d6668e79a05e756588d9e4d4be7246ab1c \ + --hash=sha256:63ddb992931a5e616c87d3d89f5a58db086e617548005c7f9059fac68c03a5cc \ + --hash=sha256:6943da9c09870490dcfd50c4909c0cc19f434fa6948f61282dc9cb07bcf08160 \ + --hash=sha256:6ad40f85c1207803d581d5d75e9ea25327cd524925699a83dfc03bf8e4ba72b7 \ + --hash=sha256:6b44433a79bdd7af0e3337bd7bbcf53dd1f9b0fa66bf21bcb756060ce32a96c1 \ + --hash=sha256:6bbaa245015d933a4172395baad7874373f162955d73612f0b66b6c2c33b6366 \ + --hash=sha256:7007227f4ea85b40a2f5e5a244479f6a6dfcf906db9b55e812a814a8f0e2c28d \ + --hash=sha256:74884a0aec1f1609190ec8b34b5d58fb3b5353cf22b96161e13e0e835f13518f \ + --hash=sha256:7d25fe5571ddb16369054f54cdd883f23de9941476d97f2b92eb6d7d83afe22d \ + --hash=sha256:7e162bdc5e3baad26b2262240be7d2bab36991d85a6a556e48b9dfb402370261 \ + --hash=sha256:814d62678dc3a30f4aa081982d830b7c342cf230ffc9d030b020cb154eeebf9e \ + --hash=sha256:8878a34c5313ee52e20aa50b03138af8d472bae465710fb954d133a9bfd3c38d \ + --hash=sha256:a66a0d94e5b081d5d695e66d6667e91e74d79e273eee95c1747717ba9cb70792 \ + --hash=sha256:a69f5cbf4addcfdf03dda564a671040127a6b7c34cf9fe4973582e68441b63fa \ + --hash=sha256:b00f9f0c334d07709d3f73a7cb8ae63c6ca1a90c790a63b5e7effa666ef96021 \ + --hash=sha256:b6ed71e4a7b4690447b626f499d978aa13197a0e592950e5d7020308f6054698 \ + --hash=sha256:bdf5041e5851526e885af579d2f455348dba68d74f14a32781933569a327fddf \ + --hash=sha256:be034360dd34e62608419f86e799c97d389c10a0e677a25f236a971b2f40dac9 \ + --hash=sha256:cc8f590a5eed30b314ae6b0232d925519ade433f663de79cc3783e4b10d662ba \ + --hash=sha256:cd7a318a15fe6cc4584bf3c4426f092ed08c0fd012cf2a9173114234fe193e11 \ + --hash=sha256:cf19b5f63a59c20306e034e691402b02055c8f4e38bf6792c23cad489162a642 \ + --hash=sha256:cfc781ce442ec407c841e9aa51d0e1024f72b6ec34caa8fdb6ef9576d549acf2 \ + --hash=sha256:dea9f6f8633571e18bc20cad83603072e697103a567f4b0738d52dd0211b4527 \ + --hash=sha256:e4a86a1d5eb2cce83c5972b3930c7c1eac81ab3508464345e2b8e54f119d5505 \ + --hash=sha256:e7106374d4a74ed9ff00c46cc00f0a9f06a0775f8868e423f85d4464d2333679 \ + --hash=sha256:e98a8a585b5668aa9e34d10f7785abf9545fe72663b4bfc16c99a115185ae6a5 \ + --hash=sha256:f64840e68483316eb58d82c376ad3585ca995e69e33b230436de0cdddf7363f9 \ + --hash=sha256:f8f4b0a9e6683e43889852130595c8854d8ae237f2324a053cdd884de936aa9b \ + --hash=sha256:fc45a53219ed30a7f670a6d8c98527af0020e6fd4ee4c0a8fb59f147f06d816c diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index f5140f9c7..10308bd39 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,7 +1,7 @@ # Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in letsencrypt-auto's requirements file. # Some dev package versions specified here may be overridden by higher level constraints -# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). +# files during tests (eg. tools/certbot_constraints.txt). alabaster==0.7.10 apacheconfig==0.3.2 apipkg==1.4 @@ -16,8 +16,8 @@ backports.functools-lru-cache==1.5 backports.shutil-get-terminal-size==1.0.0 backports.ssl-match-hostname==3.7.0.1 bcrypt==3.1.6 -boto3==1.11.7 -botocore==1.14.7 +boto3==1.17.4 +botocore==1.20.4 cached-property==1.5.1 cloudflare==2.3.1 configparser==3.7.4 diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index 0d3626853..d2ebe3537 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -14,10 +14,6 @@ WORKDIR /opt/certbot # Copy certbot code COPY CHANGELOG.md README.rst src/ -# We keep the relative path to the requirements file the same because, as of -# writing this, tools/pip_install.py is used in the Dockerfile for Certbot -# plugins and this script expects to find the requirements file there. -COPY letsencrypt-auto-source/pieces/dependency-requirements.txt letsencrypt-auto-source/pieces/ COPY tools tools COPY acme src/acme COPY certbot src/certbot diff --git a/tools/pip_install.py b/tools/pip_install.py index c1c81482b..e06650ff2 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -57,7 +57,7 @@ def certbot_oldest_processing(tools_path, args, test_constraints): def certbot_normal_processing(tools_path, test_constraints): repo_path = os.path.dirname(tools_path) certbot_requirements = os.path.normpath(os.path.join( - repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) + repo_path, 'tools/certbot_constraints.txt')) with open(certbot_requirements, 'r') as fd: certbot_reqs = fd.readlines() with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd: @@ -76,8 +76,7 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain # Here is the order by increasing priority: # 1) The general development constraints (tools/dev_constraints.txt) # 2) The general tests constraints (oldest_requirements.txt or - # certbot-auto's dependency-requirements.txt + pipstrap's constraints - # for the normal processing) + # certbot_constraints.txt + pipstrap's constraints for the normal processing) # 3) The local requirement file, typically local-oldest-requirement in oldest tests files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] if requirements: @@ -134,6 +133,7 @@ def main(args): pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"' .format(requirements)) + print(' '.join(args)) pip_install_with_print(' '.join(args), env=env) diff --git a/letsencrypt-auto-source/rebuild_dependencies.py b/tools/rebuild_certbot_constraints.py similarity index 84% rename from letsencrypt-auto-source/rebuild_dependencies.py rename to tools/rebuild_certbot_constraints.py index 864394661..f5e5d3ca7 100755 --- a/letsencrypt-auto-source/rebuild_dependencies.py +++ b/tools/rebuild_certbot_constraints.py @@ -4,12 +4,12 @@ Gather and consolidate the up-to-date dependencies available and required to ins on various Linux distributions. It generates a requirements file contained the pinned and hashed versions, ready to be used by pip to install the certbot dependencies. -This script is typically used to update the certbot-requirements.txt file of certbot-auto. +This script is typically used to update the certbot_constraints.txt file. To achieve its purpose, this script will start a certbot installation with unpinned dependencies, then gather them, on various distributions started as Docker containers. -Usage: letsencrypt-auto-source/rebuild_dependencies new_requirements.txt +Usage: tools/rebuild_certbot_constraints.py new_requirements.txt NB1: Docker must be installed on the machine running this script. NB2: Python library 'hashin' must be installed on the machine running this script. @@ -26,52 +26,41 @@ import argparse # The list of docker distributions to test dependencies against with. DISTRIBUTION_LIST = [ - 'ubuntu:18.04', 'ubuntu:16.04', - 'debian:stretch', - 'centos:7', 'centos:6', - 'opensuse/leap:15', - 'fedora:29', + 'ubuntu:20.04', 'ubuntu:18.04', 'debian:buster', + 'centos:8', 'centos:7', 'fedora:29', ] # These constraints will be added while gathering dependencies on each distribution. # It can be used because a particular version for a package is required for any reason, # or to solve a version conflict between two distributions requirements. AUTHORITATIVE_CONSTRAINTS = { - # Using an older version of mock here prevents regressions of #5276. - 'mock': '1.3.0', # Too touchy to move to a new version. And will be removed soon # in favor of pure python parser for Apache. 'python-augeas': '0.5.0', - # Package enum34 needs to be explicitly limited to Python2.x, in order to avoid - # certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456. - 'enum34': '1.1.10; python_version < \'3.4\'', - # Cryptography 2.9+ drops support for OpenSSL 1.0.1, but we still want to support it - # for officially supported non-x86_64 ancient distributions like RHEL 6. - 'cryptography': '2.8', - # Parsedatetime 2.6 is broken on Python 2.7, see https://github.com/bear/parsedatetime/issues/246 - 'parsedatetime': '2.5', + # We avoid cryptography 3.4+ since it requires Rust to compile the wheels, and + # this needs some work on the snap builds. + 'cryptography': '3.3.2', } -# ./certbot/letsencrypt-auto-source/rebuild_dependencies.py (2 levels from certbot root path) +# ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path) CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__))) # The script will be used to gather dependencies for a given distribution. -# - certbot-auto is used to install relevant OS packages, and set up an initial venv +# - bootstrap_os_packages.sh is used to install relevant OS packages, and set up an initial venv # - then this venv is used to consistently construct an empty new venv -# - once pipstraped, this new venv pip-installs certbot runtime (including apache/nginx), +# - once pipstrap.py, this new venv pip-installs certbot runtime (including apache/nginx), # without pinned dependencies, and respecting input authoritative requirements # - `certbot plugins` is called to check we have a healthy environment # - finally current set of dependencies is extracted out of the docker using pip freeze SCRIPT = r"""#!/bin/sh -set -e +set -ex cd /tmp/certbot -letsencrypt-auto-source/letsencrypt-auto --install-only -n -PYVER=`/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` +tests/letstest/scripts/bootstrap_os_packages.sh -/opt/eff.org/certbot/venv/bin/python letsencrypt-auto-source/pieces/create_venv.py /tmp/venv "$PYVER" 1 +python3 -m venv /tmp/venv -/tmp/venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py +/tmp/venv/bin/python tools/pipstrap.py /tmp/venv/bin/pip install -e acme -e certbot -e certbot-apache -e certbot-nginx -c /tmp/constraints.txt /tmp/venv/bin/certbot plugins /tmp/venv/bin/pip freeze >> /tmp/workspace/requirements.txt @@ -109,6 +98,7 @@ def _requirements_from_one_distribution(distribution, verbose): '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) command = ['docker', 'run', '--rm', '--cidfile', cid_file, + '--network=host', '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), '-v', '{0}:/tmp/workspace'.format(workspace), '-v', '{0}:/tmp/constraints.txt'.format(authoritative_constraints), @@ -158,7 +148,7 @@ def _parse_and_merge_requirements(dependencies_map, requirements_file_lines, dis """ for line in requirements_file_lines: match = re.match(r'([^=]+)==([^=]+)', line.strip()) - if not line.startswith('-e') and match: + if not line.startswith('-e') and not line.startswith('#') and match: package, version = match.groups() if package not in ['acme', 'certbot', 'certbot-apache', 'certbot-nginx', 'pkg-resources']: dependencies_map.setdefault(package, []).append((version, distribution)) @@ -215,11 +205,11 @@ def _write_requirements(dest_file, requirements, conflicts): print('===> Calculating hashes for the requirement file.') _write_to(dest_file, '''\ -# This is the flattened list of packages certbot-auto installs. +# This is the flattened list of pinned packages to build certbot deployable artifacts. # To generate this, do (with docker and package hashin installed): # ``` -# letsencrypt-auto-source/rebuild_dependencies.py \\ -# letsencrypt-auto-source/pieces/dependency-requirements.txt +# tools/rebuild_certbot_contraints.py \\ +# tools/certbot_constraints.txt # ``` # If you want to update a single dependency, run commands similar to these: # ``` @@ -264,8 +254,8 @@ def _gather_dependencies(dest_file, verbose): if __name__ == '__main__': parser = argparse.ArgumentParser( - description=('Build a sanitized, pinned and hashed requirements file for certbot-auto, ' - 'validated against several OS distributions using Docker.')) + description=('Build a sanitized, pinned and hashed requirements file for certbot deployable' + ' artifacts, validated against several OS distributions using Docker.')) parser.add_argument('requirements_path', help='path for the generated requirements file') parser.add_argument('--verbose', '-v', action='store_true', diff --git a/tools/snap/generate_dnsplugins_all.sh b/tools/snap/generate_dnsplugins_all.sh index 40404bf9b..976b0dd7b 100755 --- a/tools/snap/generate_dnsplugins_all.sh +++ b/tools/snap/generate_dnsplugins_all.sh @@ -10,7 +10,7 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH # Create constraints file "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt) \ + <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_constraints.txt) \ <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \ > "${PLUGIN_PATH}"/snap-constraints.txt done diff --git a/windows-installer/construct.py b/windows-installer/construct.py index 60834e7e5..eb199a7e1 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -2,6 +2,7 @@ import contextlib import ctypes import os +import re import shutil import struct import subprocess @@ -52,6 +53,21 @@ def _compile_wheels(repo_path, build_path, venv_python): command.extend(wheels_project) subprocess.check_call(command, env=env) + # Cryptography uses now a unique wheel name "cryptography-VERSION-cpXX-abi3-win32.whl where + # cpXX is the lowest supported version of Python (eg. cp36 says that the wheel is compatible + # with Python 3.6+). While technically valid to describe a wheel compliant with the Stable + # Application Binary Interface, this naming convention makes pynsist falsely think that the + # wheel is compatible with Python 3.6 only. + # Let's trick pynsist by renaming the wheel until this is fixed upstream. + for file in os.listdir(wheels_path): + # Given that our Python version is 3.8, this rename files like + # cryptography-VERSION-cpXX-abi3-win32.whl into cryptography-VERSION-cp38-abi3-win32.whl + renamed = re.sub(r'^(.*)-cp\d+-abi3-(\w+)\.whl$', r'\1-cp{0}{1}-abi3-\2.whl' + .format(PYTHON_VERSION[0], PYTHON_VERSION[1]), file) + print(renamed) + if renamed != file: + os.replace(os.path.join(wheels_path, file), os.path.join(wheels_path, renamed)) + def _prepare_build_tools(venv_path, venv_python, repo_path): print('Prepare build tools') @@ -63,7 +79,7 @@ def _prepare_build_tools(venv_path, venv_python, repo_path): @contextlib.contextmanager def _prepare_constraints(repo_path): - reqs_certbot = os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'dependency-requirements.txt') + reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_constraints.txt') reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt') constraints_certbot = subprocess.check_output( [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot], From ae3ed200c09f2d1913ec69c33a2604ae1fe32eb5 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 24 Feb 2021 14:51:57 -0800 Subject: [PATCH 10/39] Remove check for 'fake' in issuer name when renewing certs (#8685) Fixes #8680. We seem to have no existing testing code anywhere in this vicinity, so figured I'd get this up quickly then work on that. Manual tests (renew staging certificate, should allow it; renew non-staging cert as staging, should error) passed. * Remove check for 'fake' in issuer name when renewing certs * Change fake issuer name to make sure we're not relying on it anywhere --- certbot/certbot/_internal/renewal.py | 5 +---- certbot/tests/main_test.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index 9fe9cb546..7533c8c6b 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -312,12 +312,9 @@ def _avoid_invalidating_lineage(config, lineage, original_server): contents = the_file.read() latest_cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, contents) - # all our test certificates are from happy hacker fake CA, though maybe one day - # we should test more methodically - now_valid = "fake" not in repr(latest_cert.get_issuer()).lower() if util.is_staging(config.server): - if not util.is_staging(original_server) or now_valid: + if not util.is_staging(original_server): if not config.break_my_certs: names = ", ".join(lineage.names()) raise errors.Error( diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index ddd911c8d..785433585 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1053,7 +1053,7 @@ class MainTest(test_util.ConfigTestCase): mock_get_utility().notification.side_effect = write_msg with mock.patch('certbot._internal.main.renewal.OpenSSL') as mock_ssl: mock_latest = mock.MagicMock() - mock_latest.get_issuer.return_value = "Fake fake" + mock_latest.get_issuer.return_value = "Artificial pretend" mock_ssl.crypto.load_certificate.return_value = mock_latest with mock.patch('certbot._internal.main.renewal.crypto_util') \ as mock_crypto_util: From 025eb16c7a07b5ed3286a17c7a53bb00020b657c Mon Sep 17 00:00:00 2001 From: alexzorin Date: Fri, 26 Feb 2021 05:22:40 +1100 Subject: [PATCH 11/39] docs: rewrite "Revoking certificates" (#8657) * docs: rewrite "Revoking certificates" - `--cert-name` is supported since a long time ago - `--delete-after-revoke` is default - Mention that non-default `--server` must be specified - Document difference between acme key/cert key revocation methods - Reshuffle text to keep more important things earlier * minor edits * remove revocation note * remove "preauthorization" revocation method * rewrite deletion note --- certbot/docs/using.rst | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index ab8d64d79..1d97caecc 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -474,29 +474,37 @@ like Revoking certificates --------------------- -If your account key has been compromised or you otherwise need to revoke a certificate, -use the ``revoke`` command to do so. Note that the ``revoke`` command takes the certificate path -(ending in ``cert.pem``), not a certificate name or domain. Example:: +If you need to revoke a certificate, use the ``revoke`` subcommand to do so. - certbot revoke --cert-path /etc/letsencrypt/live/CERTNAME/cert.pem +A certificate may be revoked by providing its name (see ``certbot certificates``) or by providing +its path directly:: + + certbot revoke --cert-name example.com + + certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem + +If the certificate being revoked was obtained via the ``--staging``, ``--test-cert`` or a non-default ``--server`` flag, +that flag must be passed to the ``revoke`` subcommand. + +.. note:: After revocation, Certbot will (by default) ask whether you want to **delete** the certificate. + Unless deleted, Certbot will try to renew revoked certificates the next time ``certbot renew`` runs. You can also specify the reason for revoking your certificate by using the ``reason`` flag. Reasons include ``unspecified`` which is the default, as well as ``keycompromise``, ``affiliationchanged``, ``superseded``, and ``cessationofoperation``:: - certbot revoke --cert-path /etc/letsencrypt/live/CERTNAME/cert.pem --reason keycompromise + certbot revoke --cert-name example.com --reason keycompromise -Additionally, if a certificate -is a test certificate obtained via the ``--staging`` or ``--test-cert`` flag, that flag must be passed to the -``revoke`` subcommand. -Once a certificate is revoked (or for other certificate management tasks), all of a certificate's -relevant files can be removed from the system with the ``delete`` subcommand:: +Revoking by account key or certificate private key +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - certbot delete --cert-name example.com +By default, Certbot will try revoke the certificate using your ACME account key. If the certificate was created from +the same ACME account, the revocation will be successful. -.. note:: If you don't use ``delete`` to remove the certificate completely, it will be renewed automatically at the next renewal event. +If you instead have the corresponding private key file to the certificate you wish to revoke, use ``--key-path`` to perform the +revocation from any ACME account:: -.. note:: Revoking a certificate will have no effect on the rate limit imposed by the Let's Encrypt server. + certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem --key-path /etc/letsencrypt/live/example.com/privkey.pem .. _renewal: From f71298f6614078454532f82aeafbe5d06d3a7cdb Mon Sep 17 00:00:00 2001 From: alexzorin Date: Fri, 26 Feb 2021 06:32:21 +1100 Subject: [PATCH 12/39] cli: make key_path and cert_path always be a str (#8687) There is some code in [`_paths_parser`](https://github.com/certbot/certbot/blob/ae3ed200c09f2d1913ec69c33a2604ae1fe32eb5/certbot/certbot/_internal/cli/paths_parser.py#L17-L34) which has the effect of varying the value type of `config.cert_path` and `config.key_path` based on the CLI verb. When the verb is `revoke`, the type is a tuple `(path: str, contents: bytes)`, otherwise it is a single `str` representing the file path. (I wasn't able to find a written reason as to why it works this way). This commit removes that special `revoke` case and ensures it is always a `str`. Why change it now? I am trying to write some changes and there's some code in `cert_manager` which only works if the verb is `revoke`, you hack `config.cert_path` to be a tuple beforehand, or you [(not actually in `master`) try sniff for the value type](https://github.com/certbot/certbot/blob/49911afaa62ade1fca4c4ec407f817720cb354e3/certbot/certbot/_internal/cert_manager.py#L224-L227). I have a bad feeling about such workarounds. I would prefer to just make these variables simpler to use, but I'm open to opinions. In addition to the test suites, I've manually tested `revoke` (including by `--key-path`) and `install`. Are there other places I may have missed? Unblocks #8636 and #8671. --- certbot/certbot/_internal/cert_manager.py | 4 ++-- certbot/certbot/_internal/cli/paths_parser.py | 20 +++++++++---------- certbot/certbot/_internal/main.py | 15 ++++++++------ certbot/certbot/_internal/storage.py | 7 ++----- certbot/tests/cert_manager_test.py | 16 +++++++-------- certbot/tests/main_test.py | 7 ++----- certbot/tests/storage_test.py | 4 ++-- 7 files changed, 34 insertions(+), 39 deletions(-) diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index dfbe4b538..ee2bd6254 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -223,7 +223,7 @@ def cert_path_to_lineage(cli_config): """ acceptable_matches = _acceptable_matches() match = match_and_check_overlaps(cli_config, acceptable_matches, - lambda x: cli_config.cert_path[0], lambda x: x.lineagename) + lambda x: cli_config.cert_path, lambda x: x.lineagename) return match[0] @@ -254,7 +254,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func matched = _search_lineages(cli_config, find_matches, [], acceptable_matches) if not matched: - raise errors.Error("No match found for cert-path {0}!".format(cli_config.cert_path[0])) + raise errors.Error("No match found for cert-path {0}!".format(cli_config.cert_path)) elif len(matched) > 1: raise errors.OverlappingMatchFound() return matched diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py index 62f5e224d..04b3725b9 100644 --- a/certbot/certbot/_internal/cli/paths_parser.py +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -2,7 +2,6 @@ paths for certificates""" from certbot.compat import os from certbot._internal.cli import ( - read_file, flag_default, config_help ) @@ -14,22 +13,21 @@ def _paths_parser(helpful): if verb == "help": verb = helpful.help_arg - cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked." - sections = ["paths", "install", "revoke", "certonly", "manage"] + cpkwargs = { + "type": os.path.abspath, + "help": "Path to where certificate is saved (with certonly --csr), installed " + "from, or revoked" + } if verb == "certonly": - add(sections, "--cert-path", type=os.path.abspath, - default=flag_default("auth_cert_path"), help=cph) + cpkwargs["default"] = flag_default("auth_cert_path") elif verb == "revoke": - add(sections, "--cert-path", type=read_file, required=False, help=cph) - else: - add(sections, "--cert-path", type=os.path.abspath, help=cph) + cpkwargs["required"] = False + add(["paths", "install", "revoke", "certonly", "manage"], "--cert-path", **cpkwargs) section = "paths" if verb in ("install", "revoke"): section = verb - # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", - type=((verb == "revoke" and read_file) or os.path.abspath), + add(section, "--key-path", type=os.path.abspath, help="Path to private key for certificate installation " "or revocation (if account key is missing)") diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index b9b6b16f6..36ebb1a50 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -1098,15 +1098,18 @@ def revoke(config, unused_plugins): if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using certificate key %s", - config.cert_path[0], config.key_path[0]) - crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) - key = jose.JWK.load(config.key_path[1]) + config.cert_path, config.key_path) + crypto_util.verify_cert_matches_priv_key(config.cert_path, config.key_path) + with open(config.key_path, 'rb') as f: + key = jose.JWK.load(f.read()) acme = client.acme_from_config_key(config, key) else: # revocation by account key - logger.debug("Revoking %s using Account Key", config.cert_path[0]) + logger.debug("Revoking %s using Account Key", config.cert_path) acc, _ = _determine_account(config) acme = client.acme_from_config_key(config, acc.key, acc.regr) - cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] + + with open(config.cert_path, 'rb') as f: + cert = crypto_util.pyopenssl_load_certificate(f.read())[0] logger.debug("Reason code for revocation: %s", config.reason) try: acme.revoke(jose.ComparableX509(cert), config.reason) @@ -1114,7 +1117,7 @@ def revoke(config, unused_plugins): except acme_errors.ClientError as e: return str(e) - display_ops.success_revocation(config.cert_path[0]) + display_ops.success_revocation(config.cert_path) return None diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 690567a17..d04bc5774 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -58,7 +58,7 @@ def renewal_file_for_certname(config, certname): return path -def cert_path_for_cert_name(config, cert_name): +def cert_path_for_cert_name(config: interfaces.IConfig, cert_name: str) -> str: """ If `--cert-name` was specified, but you need a value for `--cert-path`. :param `configuration.NamespaceConfig` config: parsed command line arguments @@ -66,10 +66,7 @@ def cert_path_for_cert_name(config, cert_name): """ cert_name_implied_conf = renewal_file_for_certname(config, cert_name) - fullchain_path = configobj.ConfigObj(cert_name_implied_conf)["fullchain"] - with open(fullchain_path) as f: - cert_path = (fullchain_path, f.read()) - return cert_path + return configobj.ConfigObj(cert_name_implied_conf)["fullchain"] def config_with_defaults(config=None): diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index b26c1f624..ba6cfddc3 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -526,7 +526,7 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config): from certbot._internal.cert_manager import cert_path_to_lineage @@ -556,21 +556,21 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): mock_acceptable_matches.return_value = [lambda x: x.cert_path] test_cert_path = os.path.join(self.config.config_dir, 'live', 'example.org', 'cert.pem') - self.config.cert_path = (test_cert_path, '') + self.config.cert_path = test_cert_path self.assertEqual('example.org', self._call(self.config)) @mock.patch('certbot._internal.cert_manager._acceptable_matches') def test_options_archive_cert(self, mock_acceptable_matches): # Also this and the next test check that the regex of _archive_files is working. - self.config.cert_path = (os.path.join(self.config.config_dir, 'archive', 'example.org', - 'cert11.pem'), '') + self.config.cert_path = os.path.join(self.config.config_dir, 'archive', 'example.org', + 'cert11.pem') mock_acceptable_matches.return_value = [lambda x: self._archive_files(x, 'cert')] self.assertEqual('example.org', self._call(self.config)) @mock.patch('certbot._internal.cert_manager._acceptable_matches') def test_options_archive_fullchain(self, mock_acceptable_matches): - self.config.cert_path = (os.path.join(self.config.config_dir, 'archive', - 'example.org', 'fullchain11.pem'), '') + self.config.cert_path = os.path.join(self.config.config_dir, 'archive', + 'example.org', 'fullchain11.pem') mock_acceptable_matches.return_value = [lambda x: self._archive_files(x, 'fullchain')] self.assertEqual('example.org', self._call(self.config)) @@ -586,7 +586,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config, acceptable_matches, match_func, rv_func): from certbot._internal.cert_manager import match_and_check_overlaps @@ -595,7 +595,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): def test_basic_match(self): from certbot._internal.cert_manager import _acceptable_matches self.assertEqual(['example.org'], self._call(self.config, _acceptable_matches(), - lambda x: self.config.cert_path[0], lambda x: x.lineagename)) + lambda x: self.config.cert_path, lambda x: x.lineagename)) @mock.patch('certbot._internal.cert_manager._search_lineages') def test_no_matches(self, mock_search_lineages): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 785433585..2d5d88947 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -287,10 +287,7 @@ class RevokeTest(test_util.TempDirTestCase): super(RevokeTest, self).setUp() shutil.copy(CERT_PATH, self.tempdir) - self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, - 'cert_512.pem')) - with open(self.tmp_cert_path, 'r') as f: - self.tmp_cert = (self.tmp_cert_path, f.read()) + self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, 'cert_512.pem')) patches = [ mock.patch('acme.client.BackwardsCompatibleClientV2'), @@ -349,7 +346,7 @@ class RevokeTest(test_util.TempDirTestCase): def test_revoke_by_certname(self, mock_cert_path_for_cert_name, mock_delete_if_appropriate): args = 'revoke --cert-name=example.com'.split() - mock_cert_path_for_cert_name.return_value = self.tmp_cert + mock_cert_path_for_cert_name.return_value = self.tmp_cert_path mock_delete_if_appropriate.return_value = False self._call(args) self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index abd496c8d..0f696bc34 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -934,14 +934,14 @@ class CertPathForCertNameTest(BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config, certname): from certbot._internal.storage import cert_path_for_cert_name return cert_path_for_cert_name(cli_config, certname) def test_simple_cert_name(self): - self.assertEqual(self._call(self.config, 'example.org'), (self.fullchain, 'fullchain')) + self.assertEqual(self._call(self.config, 'example.org'), self.fullchain) def test_no_such_cert_name(self): self.assertRaises(errors.CertStorageError, self._call, self.config, 'fake-example.org') From e742cfaa21176f1f5a3e43c8022b9c899bc5c205 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Feb 2021 13:39:55 -0800 Subject: [PATCH 13/39] dont set required to False (#8689) --- certbot/certbot/_internal/cli/paths_parser.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py index 04b3725b9..43ee41c88 100644 --- a/certbot/certbot/_internal/cli/paths_parser.py +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -20,8 +20,6 @@ def _paths_parser(helpful): } if verb == "certonly": cpkwargs["default"] = flag_default("auth_cert_path") - elif verb == "revoke": - cpkwargs["required"] = False add(["paths", "install", "revoke", "certonly", "manage"], "--cert-path", **cpkwargs) section = "paths" From 135187f03e18eebcb32b05c4ecb76a4552aed883 Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Thu, 25 Feb 2021 23:50:54 +0100 Subject: [PATCH 14/39] Python 3 obsoletes explicit __ne__ methods (#8676) This shouldn't be needed as of Python 3+. https://stackoverflow.com/questions/4352244/should-ne-be-implemented-as-the-negation-of-eq-in-python#30676267 --- acme/acme/messages.py | 3 --- certbot-apache/certbot_apache/_internal/obj.py | 6 ------ 2 files changed, 9 deletions(-) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 44ecb143c..61fd89dfd 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -150,9 +150,6 @@ class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore def __hash__(self): return hash((self.__class__, self.name)) - def __ne__(self, other): - return not self == other - class Status(_Constant): """ACME "status" field.""" diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 498766744..3cd5f0ff2 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -20,9 +20,6 @@ class Addr(common.Addr): self.is_wildcard() and other.is_wildcard())) return False - def __ne__(self, other): - return not self.__eq__(other) - def __repr__(self): return "certbot_apache._internal.obj.Addr(" + repr(self.tup) + ")" @@ -191,9 +188,6 @@ class VirtualHost(object): return False - def __ne__(self, other): - return not self.__eq__(other) - def __hash__(self): return hash((self.filep, self.path, tuple(self.addrs), tuple(self.get_names()), From 67c2b27af7d7dc04c246be4f9a1d4cd29a3099ca Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Thu, 25 Feb 2021 23:59:00 +0100 Subject: [PATCH 15/39] Stop inheriting from object. It's unneeded on Python 3+. (#8675) --- acme/acme/client.py | 6 +++--- acme/acme/crypto_util.py | 6 +++--- acme/acme/magic_typing.py | 2 +- acme/acme/messages.py | 2 +- acme/acme/mixins.py | 2 +- acme/acme/standalone.py | 2 +- certbot-apache/certbot_apache/_internal/dualparser.py | 2 +- certbot-apache/certbot_apache/_internal/obj.py | 2 +- certbot-apache/certbot_apache/_internal/parser.py | 2 +- .../certbot_integration_tests/certbot_tests/context.py | 2 +- certbot-ci/certbot_integration_tests/utils/acme_server.py | 2 +- certbot-ci/certbot_integration_tests/utils/dns_server.py | 2 +- .../certbot_compatibility_test/configurators/common.py | 2 +- .../certbot_compatibility_test/validator.py | 2 +- .../certbot_dns_cloudflare/_internal/dns_cloudflare.py | 2 +- .../certbot_dns_digitalocean/_internal/dns_digitalocean.py | 2 +- .../certbot_dns_google/_internal/dns_google.py | 2 +- certbot-dns-google/tests/dns_google_test.py | 2 +- .../certbot_dns_rfc2136/_internal/dns_rfc2136.py | 2 +- certbot-nginx/certbot_nginx/_internal/nginxparser.py | 4 ++-- certbot-nginx/certbot_nginx/_internal/obj.py | 2 +- certbot-nginx/certbot_nginx/_internal/parser.py | 2 +- certbot-nginx/certbot_nginx/_internal/parser_obj.py | 2 +- certbot/certbot/_internal/account.py | 2 +- certbot/certbot/_internal/auth_handler.py | 2 +- certbot/certbot/_internal/cli/cli_utils.py | 4 ++-- certbot/certbot/_internal/cli/helpful.py | 2 +- certbot/certbot/_internal/client.py | 4 ++-- certbot/certbot/_internal/configuration.py | 2 +- certbot/certbot/_internal/display/completer.py | 2 +- certbot/certbot/_internal/error_handler.py | 2 +- certbot/certbot/_internal/lock.py | 4 ++-- certbot/certbot/_internal/plugins/disco.py | 2 +- certbot/certbot/_internal/plugins/standalone.py | 2 +- certbot/certbot/_internal/reporter.py | 2 +- certbot/certbot/display/util.py | 4 ++-- certbot/certbot/ocsp.py | 2 +- certbot/certbot/plugins/common.py | 6 +++--- certbot/certbot/plugins/dns_common.py | 2 +- certbot/certbot/plugins/dns_common_lexicon.py | 2 +- certbot/certbot/plugins/dns_test_common.py | 2 +- certbot/certbot/plugins/dns_test_common_lexicon.py | 2 +- certbot/certbot/plugins/storage.py | 2 +- certbot/certbot/reverter.py | 2 +- certbot/certbot/tests/util.py | 2 +- certbot/tests/plugins/dns_common_test.py | 2 +- tests/letstest/multitester.py | 2 +- 47 files changed, 58 insertions(+), 58 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c3f8c550f..33e515124 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -33,7 +33,7 @@ DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class ClientBase(object): +class ClientBase: """ACME client base object. :ivar messages.Directory directory: @@ -795,7 +795,7 @@ class ClientV2(ClientBase): if 'rel' in l and 'url' in l and l['rel'] == relation_type] -class BackwardsCompatibleClientV2(object): +class BackwardsCompatibleClientV2: """ACME client wrapper that tends towards V2-style calls, but supports V1 servers. @@ -938,7 +938,7 @@ class BackwardsCompatibleClientV2(object): return self.client.external_account_required() -class ClientNetwork(object): +class ClientNetwork: """Wrapper around requests that signs POSTs for authentication. Also adds user agent, and handles Content-Type. diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 4b58db328..a14737053 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore -class _DefaultCertSelection(object): +class _DefaultCertSelection: def __init__(self, certs): self.certs = certs @@ -36,7 +36,7 @@ class _DefaultCertSelection(object): return self.certs.get(server_name, None) -class SSLSocket(object): # pylint: disable=too-few-public-methods +class SSLSocket: # pylint: disable=too-few-public-methods """SSL wrapper for sockets. :ivar socket sock: Original wrapped socket. @@ -93,7 +93,7 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) - class FakeConnection(object): + class FakeConnection: """Fake OpenSSL.SSL.Connection.""" # pylint: disable=missing-function-docstring diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 388fc4a58..0e60d3203 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -2,7 +2,7 @@ import sys -class TypingClass(object): +class TypingClass: """Ignore import errors by getting anything""" def __getattr__(self, name): return None diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 61fd89dfd..0d73037ae 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -274,7 +274,7 @@ class ResourceBody(jose.JSONObjectWithFields): """ACME Resource Body.""" -class ExternalAccountBinding(object): +class ExternalAccountBinding: """ACME External Account Binding""" @classmethod diff --git a/acme/acme/mixins.py b/acme/acme/mixins.py index 1cd050ccc..0e1e0977c 100644 --- a/acme/acme/mixins.py +++ b/acme/acme/mixins.py @@ -1,7 +1,7 @@ """Useful mixins for Challenge and Resource objects""" -class VersionedLEACMEMixin(object): +class VersionedLEACMEMixin: """This mixin stores the version of Let's Encrypt's endpoint being used.""" @property def le_acme_version(self): diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 94397f0de..f5bc548b6 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -53,7 +53,7 @@ class ACMEServerMixin: allow_reuse_address = True -class BaseDualNetworkedServers(object): +class BaseDualNetworkedServers: """Base class for a pair of IPv6 and IPv4 servers that tries to do everything it's asked for both servers, but where failures in one server don't affect the other. diff --git a/certbot-apache/certbot_apache/_internal/dualparser.py b/certbot-apache/certbot_apache/_internal/dualparser.py index eef8f2a0e..1ba23e92f 100644 --- a/certbot-apache/certbot_apache/_internal/dualparser.py +++ b/certbot-apache/certbot_apache/_internal/dualparser.py @@ -4,7 +4,7 @@ from certbot_apache._internal import augeasparser from certbot_apache._internal import apacheparser -class DualNodeBase(object): +class DualNodeBase: """ Dual parser interface for in development testing. This is used as the base class for dual parser interface classes. This class handles runtime attribute value assertions.""" diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 3cd5f0ff2..e2fe48cf8 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -95,7 +95,7 @@ class Addr(common.Addr): return self.get_addr_obj(port) -class VirtualHost(object): +class VirtualHost: """Represents an Apache Virtualhost. :ivar str filep: file path of VH diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 75be0833f..fdef167bc 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -16,7 +16,7 @@ from certbot_apache._internal import constants logger = logging.getLogger(__name__) -class ApacheParser(object): +class ApacheParser: """Class handles the fine details of parsing the Apache Configuration. .. todo:: Make parsing general... remove sites-available etc... diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index b9854b402..b052e375d 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -7,7 +7,7 @@ import tempfile from certbot_integration_tests.utils import certbot_call -class IntegrationTestsContext(object): +class IntegrationTestsContext: """General fixture describing a certbot integration tests context""" def __init__(self, request): self.request = request diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index bbbdd196b..846e2d071 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -23,7 +23,7 @@ from certbot_integration_tests.utils import proxy from certbot_integration_tests.utils.constants import * -class ACMEServer(object): +class ACMEServer: """ ACMEServer configures and handles the lifecycle of an ACME CA server and an HTTP reverse proxy instance, to allow parallel execution of integration tests against the unique http-01 port diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index 416f6567e..62a58275e 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -21,7 +21,7 @@ BIND_BIND_ADDRESS = ("127.0.0.1", 45953) BIND_TEST_QUERY = bytearray.fromhex("0011cb37000000010000000000000000010003") -class DNSServer(object): +class DNSServer: """ DNSServer configures and handles the lifetime of an RFC2136-capable server. DNServer provides access to the dns_xdist parameter, listing the address and port diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index 34aa9133f..bf768f8f8 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -11,7 +11,7 @@ from certbot_compatibility_test import util logger = logging.getLogger(__name__) -class Proxy(object): +class Proxy: """A common base for compatibility test configurators""" @classmethod diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index 9a082523a..226b8585b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -10,7 +10,7 @@ from acme import errors as acme_errors logger = logging.getLogger(__name__) -class Validator(object): +class Validator: """Collection of functions to test a live webserver's configuration""" def certificate(self, cert, name, alt_host=None, port=443): diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index 5978af85c..c896a6e3a 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -85,7 +85,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _CloudflareClient(self.credentials.conf('email'), self.credentials.conf('api-key')) -class _CloudflareClient(object): +class _CloudflareClient: """ Encapsulates all communication with the Cloudflare API. """ diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index e0c9561a2..5893df9c2 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -54,7 +54,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _DigitalOceanClient(self.credentials.conf('token')) -class _DigitalOceanClient(object): +class _DigitalOceanClient: """ Encapsulates all communication with the DigitalOcean API. """ diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index 4b0d91463..363c5e079 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -76,7 +76,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _GoogleClient(self.conf('credentials')) -class _GoogleClient(object): +class _GoogleClient: """ Encapsulates all communication with the Google Cloud DNS API. """ diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 396a6c8bd..7de5f1d67 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -401,7 +401,7 @@ class GoogleClientTest(unittest.TestCase): self.assertRaises(ServerNotFoundError, _GoogleClient.get_project_id) -class DummyResponse(object): +class DummyResponse: """ Dummy object to create a fake HTTPResponse (the actual one requires a socket and we only need the status attribute) diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py index 57e9506f2..adebdd963 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py @@ -88,7 +88,7 @@ class Authenticator(dns_common.DNSAuthenticator): dns.tsig.HMAC_MD5)) -class _RFC2136Client(object): +class _RFC2136Client: """ Encapsulates all communication with the target DNS server. """ diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index f043b0e4d..a51302fae 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -19,7 +19,7 @@ from acme.magic_typing import IO, Any # pylint: disable=unused-import logger = logging.getLogger(__name__) -class RawNginxParser(object): +class RawNginxParser: # pylint: disable=pointless-statement """A class that parses nginx configuration with pyparsing.""" @@ -69,7 +69,7 @@ class RawNginxParser(object): """Returns the parsed tree as a list.""" return self.parse().asList() -class RawNginxDumper(object): +class RawNginxDumper: """A class that dumps nginx configuration from the provided tree.""" def __init__(self, blocks): self.blocks = blocks diff --git a/certbot-nginx/certbot_nginx/_internal/obj.py b/certbot-nginx/certbot_nginx/_internal/obj.py index 2dd02f180..1511cba6d 100644 --- a/certbot-nginx/certbot_nginx/_internal/obj.py +++ b/certbot-nginx/certbot_nginx/_internal/obj.py @@ -143,7 +143,7 @@ class Addr(common.Addr): return False -class VirtualHost(object): +class VirtualHost: """Represents an Nginx Virtualhost. :ivar str filep: file path of VH diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index fe5d7bb31..17a8c652a 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -21,7 +21,7 @@ from certbot_nginx._internal import obj logger = logging.getLogger(__name__) -class NginxParser(object): +class NginxParser: """Class handles the fine details of parsing the Nginx Configuration. :ivar str root: Normalized absolute path to the server root diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index d616a1a99..e55d48dc4 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -13,7 +13,7 @@ COMMENT = " managed by Certbot" COMMENT_BLOCK = ["#", COMMENT] -class Parsable(object): +class Parsable: """ Abstract base class for "Parsable" objects whose underlying representation is a tree of lists. diff --git a/certbot/certbot/_internal/account.py b/certbot/certbot/_internal/account.py index dbe111fbc..b2d50297e 100644 --- a/certbot/certbot/_internal/account.py +++ b/certbot/certbot/_internal/account.py @@ -24,7 +24,7 @@ from certbot.compat import filesystem logger = logging.getLogger(__name__) -class Account(object): +class Account: """ACME protocol registration. :ivar .RegistrationResource regr: Registration Resource diff --git a/certbot/certbot/_internal/auth_handler.py b/certbot/certbot/_internal/auth_handler.py index 7ea2a1de8..17bf75225 100644 --- a/certbot/certbot/_internal/auth_handler.py +++ b/certbot/certbot/_internal/auth_handler.py @@ -19,7 +19,7 @@ from certbot._internal import error_handler logger = logging.getLogger(__name__) -class AuthHandler(object): +class AuthHandler: """ACME Authorization Handler for a client. :ivar auth: Authenticator capable of solving diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index a0ddce38f..5bdbbe02c 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -12,7 +12,7 @@ from certbot.compat import os from certbot._internal import constants -class _Default(object): +class _Default: """A class to use as a default to detect if a value is set by a user""" def __bool__(self): @@ -66,7 +66,7 @@ def config_help(name, hidden=False): return field.__doc__ -class HelpfulArgumentGroup(object): +class HelpfulArgumentGroup: """Emulates an argparse group for use with HelpfulArgumentParser. This class is used in the add_group method of HelpfulArgumentParser. diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index 80afe5db4..5eaec978b 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -41,7 +41,7 @@ from certbot._internal.cli import ( ) -class HelpfulArgumentParser(object): +class HelpfulArgumentParser: """Argparse Wrapper. This class wraps argparse, adding the ability to make --help less diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index 5dc62580e..f2fc06937 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -93,7 +93,7 @@ def ua_flags(config): flags.append("hook") return " ".join(flags) -class DummyConfig(object): +class DummyConfig: "Shim for computing a sample user agent." def __init__(self): self.authenticator = "XXX" @@ -227,7 +227,7 @@ def perform_registration(acme, config, tos_cb): raise -class Client(object): +class Client: """Certbot's client. :ivar .IConfig config: Client configuration. diff --git a/certbot/certbot/_internal/configuration.py b/certbot/certbot/_internal/configuration.py index 1b5cf5da7..aee0022b8 100644 --- a/certbot/certbot/_internal/configuration.py +++ b/certbot/certbot/_internal/configuration.py @@ -13,7 +13,7 @@ from certbot.compat import os @zope.interface.implementer(interfaces.IConfig) -class NamespaceConfig(object): +class NamespaceConfig: """Configuration wrapper around :class:`argparse.Namespace`. For more documentation, including available attributes, please see diff --git a/certbot/certbot/_internal/display/completer.py b/certbot/certbot/_internal/display/completer.py index 03719862b..a6c984195 100644 --- a/certbot/certbot/_internal/display/completer.py +++ b/certbot/certbot/_internal/display/completer.py @@ -8,7 +8,7 @@ except ImportError: import certbot._internal.display.dummy_readline as readline # type: ignore -class Completer(object): +class Completer: """Provides Tab completion when prompting users for a path. This class is meant to be used with readline to provide Tab diff --git a/certbot/certbot/_internal/error_handler.py b/certbot/certbot/_internal/error_handler.py index 60fb287a6..05af9d837 100644 --- a/certbot/certbot/_internal/error_handler.py +++ b/certbot/certbot/_internal/error_handler.py @@ -45,7 +45,7 @@ else: _SIGNALS = [] -class ErrorHandler(object): +class ErrorHandler: """Context manager for running code that must be cleaned up on failure. The context manager allows you to register functions that will be called diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index aa0b80eaa..32142fe44 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -38,7 +38,7 @@ def lock_dir(dir_path): return LockFile(os.path.join(dir_path, '.certbot.lock')) -class LockFile(object): +class LockFile: """ Platform independent file lock system. LockFile accepts a parameter, the path to a file acting as a lock. Once the LockFile, @@ -97,7 +97,7 @@ class LockFile(object): return self._lock_mechanism.is_locked() -class _BaseLockMechanism(object): +class _BaseLockMechanism: def __init__(self, path): # type: (str) -> None """ diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index d19fdd3ef..8adf6d4d2 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -39,7 +39,7 @@ PREFIX_FREE_DISTRIBUTIONS = [ """Distributions for which prefix will be omitted.""" -class PluginEntryPoint(object): +class PluginEntryPoint: """Plugin entry point.""" # this object is mutable, don't allow it to be hashed! diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index 60c9558cb..d5d9fd2ec 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -28,7 +28,7 @@ if TYPE_CHECKING: Set[achallenges.KeyAuthorizationAnnotatedChallenge] ] -class ServerManager(object): +class ServerManager: """Standalone servers manager. Manager for `ACMEServer` and `ACMETLSServer` instances. diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index 64c0fbd6d..f43f0e7af 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IReporter) -class Reporter(object): +class Reporter: """Collects and displays information to the user. :ivar `queue.PriorityQueue` messages: Messages to be displayed to diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index 9da981892..f26ec468f 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -111,7 +111,7 @@ def notify(msg): @zope.interface.implementer(interfaces.IDisplay) -class FileDisplay(object): +class FileDisplay: """File-based display.""" # see https://github.com/certbot/certbot/issues/3915 @@ -478,7 +478,7 @@ def assert_valid_call(prompt, default, cli_flag, force_interactive): @zope.interface.implementer(interfaces.IDisplay) -class NoninteractiveDisplay(object): +class NoninteractiveDisplay: """An iDisplay implementation that never asks for interactive user input""" def __init__(self, outfile, *unused_args, **unused_kwargs): diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index b63338e2e..dcbf0381a 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -36,7 +36,7 @@ except (ImportError, AttributeError): # pragma: no cover logger = logging.getLogger(__name__) -class RevocationChecker(object): +class RevocationChecker: """This class figures out OCSP checking on this system, and performs it.""" def __init__(self, enforce_openssl_binary_usage=False): diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index d3fb5b7dc..489c75a3d 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -40,7 +40,7 @@ hostname_regex = re.compile( @zope.interface.implementer(interfaces.IPlugin) -class Plugin(object): +class Plugin: """Generic plugin.""" # provider is not inherited, subclasses must define it on their own # @zope.interface.provider(interfaces.IPluginFactory) @@ -201,7 +201,7 @@ class Installer(Plugin): constants.ALL_SSL_DHPARAMS_HASHES) -class Addr(object): +class Addr: r"""Represents an virtual host address. :param str addr: addr part of vhost address @@ -299,7 +299,7 @@ class Addr(object): return result -class ChallengePerformer(object): +class ChallengePerformer: """Abstract base for challenge performers. :ivar configurator: Authenticator and installer plugin diff --git a/certbot/certbot/plugins/dns_common.py b/certbot/certbot/plugins/dns_common.py index 245b7dc05..5c0fbcba9 100644 --- a/certbot/certbot/plugins/dns_common.py +++ b/certbot/certbot/plugins/dns_common.py @@ -233,7 +233,7 @@ class DNSAuthenticator(common.Plugin): raise errors.PluginError('{0} required to proceed.'.format(label)) -class CredentialsConfiguration(object): +class CredentialsConfiguration: """Represents a user-supplied filed which stores API credentials.""" def __init__(self, filename, mapper=lambda x: x): diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index c3d80ca29..a29509b79 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -23,7 +23,7 @@ except ImportError: logger = logging.getLogger(__name__) -class LexiconClient(object): +class LexiconClient: """ Encapsulates all communication with a DNS provider via Lexicon. """ diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index a3a26a7a1..1affcae4e 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -17,7 +17,7 @@ DOMAIN = 'example.com' KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) -class BaseAuthenticatorTest(object): +class BaseAuthenticatorTest: """ A base test class to reduce duplication between test code for DNS Authenticator Plugins. diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index 1bef06042..adeb1a268 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -34,7 +34,7 @@ class BaseLexiconAuthenticatorTest(dns_test_common.BaseAuthenticatorTest): self.assertEqual(expected, self.mock_client.mock_calls) -class BaseLexiconClientTest(object): +class BaseLexiconClientTest: DOMAIN_NOT_FOUND = Exception('No domain found') GENERIC_ERROR = RequestException LOGIN_ERROR = HTTPError('400 Client Error: ...') diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index f3ed14dce..abef534f9 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -11,7 +11,7 @@ from certbot.compat import os logger = logging.getLogger(__name__) -class PluginStorage(object): +class PluginStorage: """Class implementing storage functionality for plugins""" def __init__(self, config, classkey): diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index be9d78a11..363215dd0 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -17,7 +17,7 @@ from certbot.compat import os logger = logging.getLogger(__name__) -class Reverter(object): +class Reverter: """Reverter Class - save and revert configuration checkpoints. This class can be used by the plugins, especially Installers, to diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 558bfbed3..78236fa06 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -185,7 +185,7 @@ def patch_get_utility_with_stdout(target='zope.component.getUtility', return mock.patch(target, new=freezable_mock) -class FreezableMock(object): +class FreezableMock: """Mock object with the ability to freeze attributes. This class works like a regular mock.MagicMock object, except diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index 993f3b461..31761e986 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -29,7 +29,7 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen def more_info(self): # pylint: disable=missing-docstring,no-self-use return 'A fake authenticator for testing.' - class _FakeConfig(object): + class _FakeConfig: fake_propagation_seconds = 0 fake_config_key = 1 fake_other_key = None diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 83a92e6dc..75dd06ad6 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -95,7 +95,7 @@ SECURITY_GROUP_NAME = 'certbot-security-group' SENTINEL = None #queue kill signal SUBNET_NAME = 'certbot-subnet' -class Status(object): +class Status: """Possible statuses of client tests.""" PASS = 'pass' FAIL = 'fail' From b0e35c694e3741cb7f9c581ec25f2be1c406b53a Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Thu, 25 Feb 2021 23:59:11 +0100 Subject: [PATCH 16/39] Remove import fallback of urllib2 in tests/modification-check. (#8677) --- tests/modification-check.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/modification-check.py b/tests/modification-check.py index 7a69fb1db..357b25350 100755 --- a/tests/modification-check.py +++ b/tests/modification-check.py @@ -7,16 +7,13 @@ import shutil import subprocess import sys import tempfile +from urllib.request import urlretrieve -try: - from urllib.request import urlretrieve -except ImportError: - from urllib import urlretrieve def find_repo_path(): return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -# We do not use filecmp.cmp to take advantage of universal newlines +# We do not use filecmp.cmp to take advantage of universal newlines # handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. # As a consequence, this function will not work correctly if executed by Python 2.x on Windows. # But it will work correctly on Linux for any version, because every file tested will be LF. @@ -50,10 +47,10 @@ def validate_scripts_content(repo_path, temp_cwd): errors = True else: shutil.copyfile( - os.path.join(repo_path, 'certbot-auto'), + os.path.join(repo_path, 'certbot-auto'), os.path.join(temp_cwd, 'local-auto')) shutil.copy(os.path.normpath(os.path.join( - repo_path, + repo_path, 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) # Compare file against current version in the target branch @@ -72,7 +69,7 @@ def validate_scripts_content(repo_path, temp_cwd): latest_version = subprocess.check_output( [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) subprocess.check_call( - [sys.executable, 'fetch.py', '--le-auto-script', + [sys.executable, 'fetch.py', '--le-auto-script', 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) if compare_files( os.path.join(temp_cwd, 'letsencrypt-auto'), From 540fd6db93db317ffbeba6d210b92dd830255715 Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Fri, 26 Feb 2021 00:41:05 +0100 Subject: [PATCH 17/39] Dictionaries are ordered by insert by default on Python 3.6. (#8678) --- certbot/certbot/_internal/plugins/disco.py | 3 +-- certbot/certbot/util.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index 8adf6d4d2..1e1e00982 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -1,5 +1,4 @@ """Utilities for plugins discovery and selection.""" -import collections import itertools import logging import sys @@ -209,7 +208,7 @@ class PluginsRegistry(Mapping): # This prevents deadlock caused by plugins acquiring a lock # and ensures at least one concurrent Certbot instance will run # successfully. - self._plugins = collections.OrderedDict(sorted(plugins.items())) + self._plugins = dict(sorted(plugins.items())) @classmethod def find_all(cls): diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index 7bbc02f5f..ffd3a65f2 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -4,7 +4,6 @@ import argparse import atexit import collections -from collections import OrderedDict import distutils.version import errno import logging @@ -16,6 +15,7 @@ import sys import configargparse +from acme.magic_typing import Dict from acme.magic_typing import Text from acme.magic_typing import Tuple from acme.magic_typing import Union @@ -58,7 +58,7 @@ _INITIAL_PID = os.getpid() # the dict are attempted to be cleaned up at program exit. If the # program exits before the lock is cleaned up, it is automatically # released, but the file isn't deleted. -_LOCKS = OrderedDict() # type: OrderedDict[str, lock.LockFile] +_LOCKS = {} # type: Dict[str, lock.LockFile] def env_no_snap_for_external_calls(): From d3ca6af9824d4715f4312724e32a26c270696db8 Mon Sep 17 00:00:00 2001 From: Yuma Mihira Date: Fri, 26 Feb 2021 09:30:48 +0900 Subject: [PATCH 18/39] Insert new line before "More details about these changes can be found on our GitHub repo." (#8645) Fixing #8634. It's my first time contributing to this repository, if there's something wrong please let me know. Before this fix ``` $ python3 extract_changelog.py 1.12.0 ... ### Fixed * Fixed the apache component on openSUSE Tumbleweed which no longer provides an apache2ctl symlink and uses apachectl instead. * Fixed a typo in `certbot/crypto_util.py` causing an error upon attempting `secp521r1` key generation More details about these changes can be found on our GitHub repo. ``` After this fix ``` $ python3 extract_changelog.py 1.12.0 ... ### Fixed * Fixed the apache component on openSUSE Tumbleweed which no longer provides an apache2ctl symlink and uses apachectl instead. * Fixed a typo in `certbot/crypto_util.py` causing an error upon attempting `secp521r1` key generation More details about these changes can be found on our GitHub repo. ``` --- tools/extract_changelog.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/extract_changelog.py b/tools/extract_changelog.py index fb0b849aa..bd718191a 100755 --- a/tools/extract_changelog.py +++ b/tools/extract_changelog.py @@ -24,7 +24,7 @@ def main(): i = 0 while i < len(lines): if section_pattern.match(lines[i]): - i = i + 1 + i = i + 2 while i < len(lines): if NEW_SECTION_PATTERN.match(lines[i]): break @@ -32,8 +32,6 @@ def main(): i = i + 1 i = i + 1 - changelog = [entry for entry in changelog if entry] - print('\n'.join(changelog)) From fab9bfd878952baf98c5e3098db5f975399e0479 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Sat, 27 Feb 2021 08:43:22 +1100 Subject: [PATCH 19/39] nginx: authenticate all matching vhosts for HTTP01 (#8663) * nginx: authenticate all matching vhosts for HTTP01 Previously, the nginx authenticator would set up the HTTP-01 challenge response on a single HTTP vhost which matched the challenge domain. The nginx authenticator will now set the challenge response on every vhost which matches the challenge domain, including duplicates and HTTPS vhosts. This makes the authenticator usable behind a CDN where all origin traffic is performed over HTTPS and also makes the authenticator work more reliably against "invalid" nginx configurations, such as those where there are duplicate vhosts. * some typos * dont authenticate the same vhost twice One vhost may appear in both the HTTP and HTTPS vhost lists. Use a set() to avoid trying to mod the same vhost twice. * fix type annotations * rewrite changelog entry --- .../certbot_nginx/_internal/configurator.py | 94 ++++++++++++++----- .../certbot_nginx/_internal/http_01.py | 54 ++++++----- .../certbot_nginx/_internal/parser.py | 6 +- certbot-nginx/tests/configurator_test.py | 18 +++- certbot-nginx/tests/http_01_test.py | 45 ++++++++- certbot-nginx/tests/parser_test.py | 5 +- .../testdata/etc_nginx/sites-enabled/both.com | 32 +++++++ certbot/CHANGELOG.md | 2 + 8 files changed, 197 insertions(+), 59 deletions(-) create mode 100644 certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 15fbe61f7..8a3b8078f 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-lines """Nginx Configuration""" from distutils.version import LooseVersion import logging @@ -15,8 +16,10 @@ from acme import challenges from acme import crypto_util as acme_crypto_util from acme.magic_typing import Dict from acme.magic_typing import List +from acme.magic_typing import Optional from acme.magic_typing import Set from acme.magic_typing import Text +from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -105,7 +108,7 @@ class NginxConfigurator(common.Installer): self.save_notes = "" # For creating new vhosts if no names match - self.new_vhost = None + self.new_vhost: Optional[obj.VirtualHost] = None # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() @@ -116,7 +119,7 @@ class NginxConfigurator(common.Installer): self._chall_out = 0 # These will be set in the prepare function - self.parser = None + self.parser: Optional[parser.NginxParser] = None self.version = version self.openssl_version = openssl_version self._enhance_func = {"redirect": self._enable_redirect, @@ -377,10 +380,13 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain, allow_port_mismatch, port): + def _vhost_from_duplicated_default(self, domain: str, allow_port_mismatch: bool, port: str + ) -> obj.VirtualHost: """if allow_port_mismatch is False, only server blocks with matching ports will be used as a default server block template. """ + assert self.parser is not None # prepare should already have been called here + if self.new_vhost is None: default_vhost = self._get_default_vhost(domain, allow_port_mismatch, port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, @@ -509,7 +515,7 @@ class NginxConfigurator(common.Installer): match['rank'] += NO_SSL_MODIFIER return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): + def choose_redirect_vhosts(self, target_name: str, port: str) -> List[obj.VirtualHost]: """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -523,9 +529,6 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param str port: port number - :param bool create_if_no_match: If we should create a new vhost from default - when there is no match found. If we can't choose a default, raise a - MisconfigurationError. :returns: vhosts associated with name :rtype: list of :class:`~certbot_nginx._internal.obj.VirtualHost` @@ -538,32 +541,75 @@ class NginxConfigurator(common.Installer): else: matches = self._get_redirect_ranked_matches(target_name, port) vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] - if not vhosts and create_if_no_match: - vhosts = [self._vhost_from_duplicated_default(target_name, False, port)] return vhosts - def _port_matches(self, test_port, matching_port): + def choose_auth_vhosts(self, target_name: str) -> Tuple[List[obj.VirtualHost], + List[obj.VirtualHost]]: + """Returns a list of HTTP and HTTPS vhosts with a server_name matching target_name. + + If no HTTP vhost exists, one will be cloned from the default vhost. If that fails, no HTTP + vhost will be returned. + + :param str target_name: non-wildcard domain name + + :returns: tuple of HTTP and HTTPS virtualhosts + :rtype: tuple of :class:`~certbot_nginx._internal.obj.VirtualHost` + + """ + vhosts = [m['vhost'] for m in self._get_ranked_matches(target_name) if m and 'vhost' in m] + http_vhosts = [vh for vh in vhosts if + self._vhost_listening(vh, str(self.config.http01_port), False)] + https_vhosts = [vh for vh in vhosts if + self._vhost_listening(vh, str(self.config.https_port), True)] + + # If no HTTP vhost matches, try create one from the default_server on http01_port. + if not http_vhosts: + try: + http_vhosts = [self._vhost_from_duplicated_default(target_name, False, + str(self.config.http01_port))] + except errors.MisconfigurationError: + http_vhosts = [] + + return http_vhosts, https_vhosts + + def _port_matches(self, test_port: str, matching_port: str) -> bool: # test_port is a number, matching is a number or "" or None if matching_port == "" or matching_port is None: # if no port is specified, Nginx defaults to listening on port 80. return test_port == self.DEFAULT_LISTEN_PORT return test_port == matching_port - def _vhost_listening_on_port_no_ssl(self, vhost, port): - found_matching_port = False - if not vhost.addrs: - # if there are no listen directives at all, Nginx defaults to - # listening on port 80. - found_matching_port = (port == self.DEFAULT_LISTEN_PORT) - else: - for addr in vhost.addrs: - if self._port_matches(port, addr.get_port()) and not addr.ssl: - found_matching_port = True + def _vhost_listening(self, vhost: obj.VirtualHost, port: str, ssl: bool) -> bool: + """Tests whether a vhost has an address listening on a port with SSL enabled or disabled. - if found_matching_port: - # make sure we don't have an 'ssl on' directive - return not self.parser.has_ssl_on_directive(vhost) - return False + :param `obj.VirtualHost` vhost: The vhost whose addresses will be tested + :param port str: The port number as a string that the address should be bound to + :param bool ssl: Whether SSL should be enabled or disabled on the address + + :returns: Whether the vhost has an address listening on the port and protocol. + :rtype: bool + + """ + assert self.parser is not None # prepare should already have been called here + + # if the 'ssl on' directive is present on the vhost, all its addresses have SSL enabled + all_addrs_are_ssl = self.parser.has_ssl_on_directive(vhost) + + # if we want ssl vhosts: either 'ssl on' or 'addr.ssl' should be enabled + # if we want plaintext vhosts: neither 'ssl on' nor 'addr.ssl' should be enabled + _ssl_matches = lambda addr: addr.ssl or all_addrs_are_ssl if ssl else \ + not addr.ssl and not all_addrs_are_ssl + + # if there are no listen directives at all, Nginx defaults to + # listening on port 80. + if not vhost.addrs: + return port == self.DEFAULT_LISTEN_PORT and ssl == all_addrs_are_ssl + + return any(self._port_matches(port, addr.get_port()) and _ssl_matches(addr) + for addr in vhost.addrs) + + def _vhost_listening_on_port_no_ssl(self, vhost: obj.VirtualHost, port: str) -> bool: + return self._vhost_listening(vhost, port, False) def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index 40f994988..896760fc4 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -5,6 +5,8 @@ import logging from acme import challenges from acme.magic_typing import List +from acme.magic_typing import Optional +from certbot import achallenges from certbot import errors from certbot.compat import os from certbot.plugins import common @@ -138,13 +140,12 @@ class NginxHttp01(common.ChallengePerformer): def _get_validation_path(self, achall): return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token")) - def _make_server_block(self, achall): + def _make_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge) -> List: """Creates a server block for a challenge. + :param achall: Annotated HTTP-01 challenge - :type achall: - :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` - :param list addrs: addresses of challenged domain - :class:`list` of type :class:`~nginx.obj.Addr` + :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :returns: server block for the challenge host :rtype: list """ @@ -172,34 +173,35 @@ class NginxHttp01(common.ChallengePerformer): return location_directive - def _make_or_mod_server_block(self, achall): - """Modifies a server block to respond to a challenge. + def _make_or_mod_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge + ) -> Optional[List]: + """Modifies server blocks to respond to a challenge. Returns a new HTTP server block + to add to the configuration if an existing one can't be found. :param achall: Annotated HTTP-01 challenge - :type achall: - :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + + :returns: new server block to be added, if any + :rtype: list """ - try: - vhosts = self.configurator.choose_redirect_vhosts(achall.domain, - '%i' % self.configurator.config.http01_port, create_if_no_match=True) - except errors.MisconfigurationError: + http_vhosts, https_vhosts = self.configurator.choose_auth_vhosts(achall.domain) + + new_vhost: Optional[list] = None + if not http_vhosts: # Couldn't find either a matching name+port server block # or a port+default_server block, so create a dummy block - return self._make_server_block(achall) + new_vhost = self._make_server_block(achall) - # len is max 1 because Nginx doesn't authenticate wildcards - # if len were or vhosts None, we would have errored - vhost = vhosts[0] + # Modify any existing server blocks + for vhost in set(http_vhosts + https_vhosts): + location_directive = [self._location_directive_for_achall(achall)] - # Modify existing server block - location_directive = [self._location_directive_for_achall(achall)] + self.configurator.parser.add_server_directives(vhost, location_directive) - self.configurator.parser.add_server_directives(vhost, - location_directive) + rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', + ' ', '$1', ' ', 'break']] + self.configurator.parser.add_server_directives(vhost, + rewrite_directive, insert_at_top=True) - rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', - ' ', '$1', ' ', 'break']] - self.configurator.parser.add_server_directives(vhost, - rewrite_directive, insert_at_top=True) - return None + return new_vhost diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 17a8c652a..db9530104 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -10,6 +10,7 @@ import pyparsing from acme.magic_typing import Dict from acme.magic_typing import List +from acme.magic_typing import Optional from acme.magic_typing import Set from acme.magic_typing import Tuple from acme.magic_typing import Union @@ -361,8 +362,9 @@ class NginxParser: except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) - def duplicate_vhost(self, vhost_template, remove_singleton_listen_params=False, - only_directives=None): + def duplicate_vhost(self, vhost_template: obj.VirtualHost, + remove_singleton_listen_params: bool = False, + only_directives: Optional[List] = None) -> obj.VirtualHost: """Duplicate the vhost in the configuration files. :param :class:`~certbot_nginx._internal.obj.VirtualHost` vhost_template: The vhost diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 5e788e394..9ccc3fc9e 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -39,7 +39,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(12, len(self.config.parser.parsed)) + self.assertEqual(13, len(self.config.parser.parsed)) @mock.patch("certbot_nginx._internal.configurator.util.exe_exists") @mock.patch("certbot_nginx._internal.configurator.subprocess.Popen") @@ -89,7 +89,7 @@ class NginxConfiguratorTest(util.NginxTest): "155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", - "headers.com", "example.net"}) + "headers.com", "example.net", "ssl.both.com"}) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -935,7 +935,19 @@ class NginxConfiguratorTest(util.NginxTest): prefer_ssl=False, no_ssl_filter_port='80') # Check that the dialog was called with only port 80 vhosts - self.assertEqual(len(mock_select_vhs.call_args[0][0]), 6) + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 8) + + def test_choose_auth_vhosts(self): + """choose_auth_vhosts correctly selects duplicative and HTTP/HTTPS vhosts""" + http, https = self.config.choose_auth_vhosts('ssl.both.com') + self.assertEqual(len(http), 4) + self.assertEqual(len(https), 2) + self.assertEqual(http[0].names, {'ssl.both.com'}) + self.assertEqual(http[1].names, {'ssl.both.com'}) + self.assertEqual(http[2].names, {'ssl.both.com'}) + self.assertEqual(http[3].names, {'*.both.com'}) + self.assertEqual(https[0].names, {'ssl.both.com'}) + self.assertEqual(https[1].names, {'*.both.com'}) class InstallSslOptionsConfTest(util.NginxTest): diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index 2947b099d..d0e84fc83 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -44,6 +44,10 @@ class HttpPerformTest(util.NginxTest): challb=acme_util.chall_to_challb( challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), domain="migration.com", account_key=account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), + domain="ipv6ssl.com", account_key=account_key), ] def setUp(self): @@ -77,8 +81,8 @@ class HttpPerformTest(util.NginxTest): http_responses = self.http01.perform() - self.assertEqual(len(http_responses), 4) - for i in range(4): + self.assertEqual(len(http_responses), 5) + for i in range(5): self.assertEqual(http_responses[i], acme_responses[i]) def test_mod_config(self): @@ -105,6 +109,43 @@ class HttpPerformTest(util.NginxTest): # self.assertEqual(vhost.addrs, set(v_addr2_print)) # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')])) + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_http_and_https(self, mock_add_server_directives): + """A server_name with both HTTP and HTTPS vhosts should get modded in both vhosts""" + self.configuration.https_port = 443 + self.http01.add_chall(self.achalls[3]) # migration.com + self.http01._mod_config() # pylint: disable=protected-access + + # Domain has an HTTP and HTTPS vhost + # 2 * 'rewrite' + 2 * 'return 200 keyauthz' = 4 + self.assertEqual(mock_add_server_directives.call_count, 4) + + @mock.patch('certbot_nginx._internal.parser.nginxparser.dump') + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_only_https(self, mock_add_server_directives, mock_dump): + """A server_name with only an HTTPS vhost should get modded""" + self.http01.add_chall(self.achalls[4]) # ipv6ssl.com + self.http01._mod_config() # pylint: disable=protected-access + + # It should modify the existing HTTPS vhost + self.assertEqual(mock_add_server_directives.call_count, 2) + # since there was no suitable HTTP vhost or default HTTP vhost, a non-empty one + # should have been created and written to the challenge conf file + self.assertNotEqual(mock_dump.call_args[0][0], []) + + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_deduplicate(self, mock_add_server_directives): + """A vhost that appears in both HTTP and HTTPS vhosts only gets modded once""" + achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), + domain="ssl.both.com", account_key=AUTH_KEY) + self.http01.add_chall(achall) + self.http01._mod_config() # pylint: disable=protected-access + + # Should only get called 5 times, rather than 6, because two vhosts are the same + self.assertEqual(mock_add_server_directives.call_count, 5*2) + @mock.patch("certbot_nginx._internal.configurator.NginxConfigurator.ipv6_info") def test_default_listen_addresses_no_memoization(self, ipv6_info): # pylint: disable=protected-access diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 0083c2448..23fe390ad 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -51,6 +51,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual({nparser.abs_path(x) for x in ['foo.conf', 'nginx.conf', 'server.conf', 'sites-enabled/default', + 'sites-enabled/both.com', 'sites-enabled/example.com', 'sites-enabled/headers.com', 'sites-enabled/migration.com', @@ -88,7 +89,7 @@ class NginxParserTest(util.NginxTest): parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(9, len( + self.assertEqual(10, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -171,7 +172,7 @@ class NginxParserTest(util.NginxTest): '*.www.example.com'}, [], [2, 1, 0]) - self.assertEqual(14, len(vhosts)) + self.assertEqual(19, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] diff --git a/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com b/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com new file mode 100644 index 000000000..23a660df3 --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com @@ -0,0 +1,32 @@ +server { + server_name ssl.both.com; +} + +# a duplicate vhost +server { + server_name ssl.both.com; +} + +# a duplicate by means of wildcard +server { + server_name *.both.com; +} + +# combined HTTP and HTTPS +server { + server_name ssl.both.com; + listen 80; + listen 5001 ssl; + + ssl_certificate cert.pem; + ssl_certificate_key cert.key; +} + +# HTTPS, duplicate by means of wildcard +server { + server_name *.both.com; + listen 5001 ssl; + + ssl_certificate cert.pem; + ssl_certificate_key cert.key; +} diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 1067c3b92..ea9c35e1d 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -21,6 +21,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). Python 2. * Certbot and all of its components no longer depend on the library `six`. * The update of certbot-auto itself is now disabled on all RHEL-like systems. +* The nginx authenticator now configures all matching HTTP and HTTPS vhosts for the HTTP-01 + challenge. It is now compatible with external HTTPS redirection by a CDN or load balancer. ### Fixed From 496a4ced2560686bf5dc6769c90406ea66fea4e5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 26 Feb 2021 16:05:34 -0800 Subject: [PATCH 20/39] Remove broken test for typing import failure (#8692) * remove broken test * fix coverage * don't worry about getattr test --- acme/acme/magic_typing.py | 20 +++++++++----------- acme/tests/magic_typing_test.py | 13 ------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 0e60d3203..91308fef6 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -1,16 +1,14 @@ -"""Shim class to not have to depend on typing module in prod.""" -import sys +"""Simple shim around the typing module. + +This was useful when this code supported Python 2 and typing wasn't always +available. This code is being kept for now for backwards compatibility. + +""" +from typing import * # pylint: disable=wildcard-import, unused-wildcard-import +from typing import Collection, IO # type: ignore class TypingClass: """Ignore import errors by getting anything""" def __getattr__(self, name): - return None - -try: - # mypy doesn't respect modifying sys.modules - from typing import * # pylint: disable=wildcard-import, unused-wildcard-import - from typing import Collection, IO # type: ignore -except ImportError: - # mypy complains because TypingClass is not a module - sys.modules[__name__] = TypingClass() # type: ignore + return None # pragma: no cover diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 048995916..257164d77 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -22,19 +22,6 @@ class MagicTypingTest(unittest.TestCase): del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing - def test_import_failure(self): - try: - import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover - sys.modules['typing'] = None - if 'acme.magic_typing' in sys.modules: - del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text - self.assertTrue(Text is None) - del sys.modules['acme.magic_typing'] - sys.modules['typing'] = temp_typing - if __name__ == '__main__': unittest.main() # pragma: no cover From 1e307230038ba0e92dfe06ecfef4e22d135370d4 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Tue, 2 Mar 2021 07:56:22 +1100 Subject: [PATCH 21/39] revoke: try determine the server automatically (#8691) * revoke: try determine the server automatically When revoking via --cert-name, use the server from the lineage (unless overriden by the CLI). * RenewableCert.storage might be None * guard against an empty lineage server --- certbot/CHANGELOG.md | 2 ++ certbot/certbot/_internal/main.py | 11 +++++--- certbot/certbot/_internal/storage.py | 13 ++++++--- certbot/tests/main_test.py | 40 +++++++++++++++++++++++++--- certbot/tests/storage_test.py | 7 +++++ 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index ea9c35e1d..49bd2f222 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -21,6 +21,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). Python 2. * Certbot and all of its components no longer depend on the library `six`. * The update of certbot-auto itself is now disabled on all RHEL-like systems. +* When revoking a certificate by `--cert-name`, it is no longer necessary to specify the `--server` + if the certificate was obtained from a non-default ACME server. * The nginx authenticator now configures all matching HTTP and HTTPS vhosts for the HTTP-01 challenge. It is now compatible with external HTTPS redirection by a CDN or load balancer. diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 36ebb1a50..f7455db96 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -1072,8 +1072,7 @@ def certificates(config, unused_plugins): cert_manager.certificates(config) -# TODO: coop with renewal config -def revoke(config, unused_plugins): +def revoke(config, unused_plugins: plugins_disco.PluginsRegistry) -> Optional[str]: """Revoke a previously obtained certificate. :param config: Configuration object @@ -1090,7 +1089,13 @@ def revoke(config, unused_plugins): config.installer = config.authenticator = None if config.cert_path is None and config.certname: - config.cert_path = storage.cert_path_for_cert_name(config, config.certname) + # When revoking via --cert-name, take the cert path and server from renewalparams + lineage = storage.RenewableCert( + storage.renewal_file_for_certname(config, config.certname), config) + config.cert_path = lineage.cert_path + # --server takes priority over lineage.server + if lineage.server and not cli.set_by_cli("server"): + config.server = lineage.server elif not config.cert_path or (config.cert_path and config.certname): # intentionally not supporting --cert-path & --cert-name together, # to avoid dealing with mismatched values diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index d04bc5774..218a5dd92 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -6,6 +6,7 @@ import re import shutil import stat +from typing import Optional import configobj import parsedatetime import pytz @@ -518,11 +519,15 @@ class RenewableCert(interfaces.RenewableCert): return _relpath_from_file(self.archive_dir, from_file) @property - def is_test_cert(self): + def server(self) -> Optional[str]: + """Returns the ACME server associated with this certificate""" + return self.configuration["renewalparams"].get("server", None) + + @property + def is_test_cert(self) -> bool: """Returns true if this is a test cert from a staging server.""" - server = self.configuration["renewalparams"].get("server", None) - if server: - return util.is_staging(server) + if self.server: + return util.is_staging(self.server) return False def _check_symlinks(self): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 2d5d88947..5ad11b120 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -317,6 +317,7 @@ class RevokeTest(test_util.TempDirTestCase): if not args: args = 'revoke --cert-path={0} ' args = args.format(self.tmp_cert_path).split() + cli.set_by_cli.detector = None # required to reset set_by_cli state plugins = disco.PluginsRegistry.find_all() config = configuration.NamespaceConfig( cli.prepare_and_parse_args(plugins, args)) @@ -342,13 +343,44 @@ class RevokeTest(test_util.TempDirTestCase): self.assertEqual(expected, mock_revoke.call_args_list) @mock.patch('certbot._internal.main._delete_if_appropriate') - @mock.patch('certbot._internal.storage.cert_path_for_cert_name') - def test_revoke_by_certname(self, mock_cert_path_for_cert_name, - mock_delete_if_appropriate): + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, + server="https://acme.example") args = 'revoke --cert-name=example.com'.split() - mock_cert_path_for_cert_name.return_value = self.tmp_cert_path mock_delete_if_appropriate.return_value = False self._call(args) + self.mock_acme_client.assert_called_once_with(mock.ANY, mock.ANY, 'https://acme.example') + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + + @mock.patch('certbot._internal.main._delete_if_appropriate') + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname_and_server(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + """Revoking with --server should use the server from the CLI""" + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, + server="https://acme.example") + args = 'revoke --cert-name=example.com --server https://other.example'.split() + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_acme_client.assert_called_once_with(mock.ANY, mock.ANY, 'https://other.example') + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + + @mock.patch('certbot._internal.main._delete_if_appropriate') + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname_empty_server(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + """Revoking with --cert-name where the lineage server is empty shouldn't crash """ + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, server=None) + args = 'revoke --cert-name=example.com'.split() + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_acme_client.assert_called_once_with( + mock.ANY, mock.ANY, constants.CLI_DEFAULTS['server']) self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) @mock.patch('certbot._internal.main._delete_if_appropriate') diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 0f696bc34..1e3a1ffd8 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -765,6 +765,13 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(storage.add_time_interval(base_time, interval), excepted) + def test_server(self): + self.test_rc.configuration["renewalparams"] = {} + self.assertEqual(self.test_rc.server, None) + rp = self.test_rc.configuration["renewalparams"] + rp["server"] = "https://acme.example/dir" + self.assertEqual(self.test_rc.server, "https://acme.example/dir") + def test_is_test_cert(self): self.test_rc.configuration["renewalparams"] = {} rp = self.test_rc.configuration["renewalparams"] From 976068b5a09a4856d95dc9e408d5578b6058b963 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 2 Mar 2021 13:37:04 -0800 Subject: [PATCH 22/39] Update changelog for 1.13.0 release --- certbot/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 49bd2f222..d007166b8 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.13.0 - master +## 1.13.0 - 2021-03-02 ### Added From 92a66454b6ece097a987621031b16b1fb2f490fd Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 2 Mar 2021 13:49:58 -0800 Subject: [PATCH 23/39] Release 1.13.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 27 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- certbot/docs/cli-help.txt | 19 ++---------- letsencrypt-auto | 27 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 26 ++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 26 files changed, 83 insertions(+), 94 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 847ca9299..deaf56b59 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f129343b3..b63ed1d08 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 002fd5ffc..24af0b4c3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.12.0" +LE_AUTO_VERSION="1.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -806,6 +806,7 @@ elif [ -f /etc/mageia-release ]; then NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -1487,18 +1488,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index af19b126e..94bde0ae2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index eab6cdb70..86d4a7855 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 83513ef7c..6fa117a15 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 6e60444cf..9570bd000 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index f1fcfd11d..6a3ff1235 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 185048a2d..ebde007c6 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 0ae9c1bf7..f8bb147da 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index b16d014c6..45599b4c0 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 21ccf9d42..e6c5db942 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 2312d6fcc..831f4d0ad 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 658027b9a..4da5dbdf0 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index b4f73ddb4..b65548ec4 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index ce74611cd..a90a6ce2c 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8def9a702..27663ed28 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 6f4f8e506..aa106037f 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 385f4cc17..5c2a8ba12 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index be06b5803..5fb51f98e 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.13.0.dev0' +__version__ = '1.13.0' diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index 4482ea439..27511777b 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -118,7 +118,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.12.0 + "". (default: CertbotACMEClient/1.13.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -167,19 +167,6 @@ automation: --duplicate Allow making a certificate lineage that duplicates an existing one (both can be renewed in parallel) (default: False) - --os-packages-only (certbot-auto only) install OS package dependencies - and then stop (default: False) - --no-self-upgrade (certbot-auto only) prevent the certbot-auto script - from upgrading itself to newer released versions - (default: Upgrade automatically) - --no-bootstrap (certbot-auto only) prevent the certbot-auto script - from installing OS-level dependencies (default: Prompt - to install OS-wide dependencies, but exit if the user - says 'No') - --no-permissions-check - (certbot-auto only) skip the check on the file system - permissions of the certbot-auto script (default: - False) -q, --quiet Silence all output except errors. Useful for automation via cron. Implies --non-interactive. (default: False) @@ -254,8 +241,8 @@ paths: Flags for changing execution paths & servers --cert-path CERT_PATH - Path to where certificate is saved (with auth --csr), - installed from, or revoked. (default: None) + Path to where certificate is saved (with certonly + --csr), installed from, or revoked (default: None) --key-path KEY_PATH Path to private key for certificate installation or revocation (if account key is missing) (default: None) --fullchain-path FULLCHAIN_PATH diff --git a/letsencrypt-auto b/letsencrypt-auto index 002fd5ffc..24af0b4c3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.12.0" +LE_AUTO_VERSION="1.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -806,6 +806,7 @@ elif [ -f /etc/mageia-release ]; then NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -1487,18 +1488,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index aba5f1140..fbd16f2b0 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmAZorcACgkQTRfJlc2X -dfI6Ogf+LFASyH9sgTV1k9hs1zbmO3CxyE9QQs1JLXpoKOQ1tKv+v+kpt+lJ005g -rielyRSssXtZSyfLchCSBh6qaEBodoOcz8RS2z7rDnR9jKOJv252Buh2oSa3KPmn -WPjRmB3zVXnhq/XmPKQTnoflUlBg+MtZuZXt0Fvu8rvQB+RY3AUfB5Xs83nxJNj4 -W9qNpZYl0sJWWiydr23bEk35MJSt62sKDvyqIVjUfgDfXHmauOpg0foz2xS6XP8i -Ke66GUKaQ1ap2BTucwVT0hieXiQZpxx1PitUeEOjOH9PUfrAxyFlQ0XQaVlqoBhc -YM3nzJw9yf12b+XCUvMzHyQmDA5vdQ== -=AUGt +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmA+sv4ACgkQTRfJlc2X +dfLG+ggAvwUCqy06UZd8jZhOZFUoi8nWHG2TMnm/4CW4G9PPCjsCoQplhaAaUCrR +PAv0vtsG1rBKWekICg6IBTzLVioH9xRPUkpfbVQhT1c1T/5CqMsFFXR5p9YAKRe7 +hlOb7VRN10bdCS4JThRPNhdWFdWKZXYKcIOObWA/FX2GacxMHuLwPpSsbt2NRffy +qz1ZOWxvr289aWEbZWfyBiI2bxQ7wlMEbZ/JLUXDe46ETDxzENu+c0x2109ha6m4 +wHmUS0r16ps/n9DueTZGJf3C26mU+cIB+LgNvOcibBo+0Ly7t+OiBHYbkFXS2KTQ +RbH4bPZrsduUOzhE8wIsSUIsVGDleQ== +=jYnL -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 224abaf01..24af0b4c3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.13.0.dev0" +LE_AUTO_VERSION="1.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1488,18 +1488,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index ac143de5133fe67f7e2603a9be2f33665e78fa2c..3516f77b0476e8fcd96d806f6cb7a0714311bd12 100644 GIT binary patch literal 256 zcmV+b0ssDA%!<&T2#4iP{>&0k}(o_PX)s>Y- zrBV%mfP)is0sH4KmfDZW=Yt9<@Ql0GR>?6n=DPyOQX*xB(d{5QtnX4nS95FE%`Jww z=bnPK2Z=HKU`Gn4;8|WeAKEMOKcX;|~NR_4j^#_pW5KEftEPHR&7d6>tM z25sS8!q>0X2i1!j^>r7H3#_3h+9Hd?a0>T0w3bDqFtv1&?YEiLe&nlJm$Hh)Kyg6) zgqN!!#FtpuwhEdp+myusw`M`ny<6n)L%{I6g57Roqh20pjWc`Ewg6gn`Ye-p$Q(?^ zbC^pt$wV}Dv&JQ!aJ+c?Nd9eta%CF`q+j!a9ju|3PdE}u_D5)b8cm|&VKzf+ Date: Tue, 2 Mar 2021 13:50:03 -0800 Subject: [PATCH 24/39] Add contents to certbot/CHANGELOG.md for next version --- certbot/CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index d007166b8..7e877574f 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,6 +2,22 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 1.14.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + ## 1.13.0 - 2021-03-02 ### Added From 9d97be3a84c963c50f144cfbfc6e174212343e0a Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 2 Mar 2021 13:50:04 -0800 Subject: [PATCH 25/39] Bump version to 1.14.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index deaf56b59..90ab5c9f3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index b63ed1d08..43165a9dd 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 94bde0ae2..3e9e6a7c2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 86d4a7855..2e5215f9c 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 6fa117a15..44bada6bc 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 9570bd000..763961e79 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 6a3ff1235..99a3c92a3 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index ebde007c6..b126b4e8f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index f8bb147da..cada7a85e 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 45599b4c0..e8b26eceb 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index e6c5db942..63da31df2 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 831f4d0ad..61b9dfa78 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 4da5dbdf0..d60412f27 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index b65548ec4..4d6131ff7 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index a90a6ce2c..18d006732 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 27663ed28..fdc931bbe 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index aa106037f..55f140d76 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 5c2a8ba12..8b638f28e 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 5fb51f98e..790443bcc 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.13.0' +__version__ = '1.14.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 24af0b4c3..7f1987439 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.13.0" +LE_AUTO_VERSION="1.14.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From a3abcc001a1dff41a6ecd90f779cf038e2f32cc9 Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Wed, 3 Mar 2021 22:10:56 +0100 Subject: [PATCH 26/39] Removed a Python 2 fallback in certbot.Reverter. (#8694) * Removed a Python 2 fallback in certbot.Reverter. * Removed a Python < 3.6 fallback in certbot-apache._internal.parser. --- certbot-apache/certbot_apache/_internal/parser.py | 5 ----- certbot/certbot/reverter.py | 8 +++----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index fdef167bc..a1ea02b18 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -3,8 +3,6 @@ import copy import fnmatch import logging import re -import sys - from acme.magic_typing import Dict from acme.magic_typing import List @@ -737,9 +735,6 @@ class ApacheParser: :rtype: str """ - if sys.version_info < (3, 6): - # This strips off final /Z(?ms) - return fnmatch.translate(clean_fn_match)[:-7] # pragma: no cover # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z return fnmatch.translate(clean_fn_match)[4:-3] # pragma: no cover diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index 363215dd0..e6777a7da 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -3,7 +3,6 @@ import csv import glob import logging import shutil -import sys import time import traceback @@ -251,11 +250,10 @@ class Reverter: def _run_undo_commands(self, filepath): """Run all commands in a file.""" - # NOTE: csv module uses native strings. That is, bytes on Python 2 and - # unicode on Python 3 + # NOTE: csv module uses native strings. That is unicode on Python 3 # It is strongly advised to set newline = '' on Python 3 with CSV, # and it fixes problems on Windows. - kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + kwargs = {'newline': ''} with open(filepath, 'r', **kwargs) as csvfile: # type: ignore csvreader = csv.reader(csvfile) for command in reversed(list(csvreader)): @@ -354,7 +352,7 @@ class Reverter: command_file = None # It is strongly advised to set newline = '' on Python 3 with CSV, # and it fixes problems on Windows. - kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + kwargs = {'newline': ''} try: if os.path.isfile(commands_fp): command_file = open(commands_fp, "a", **kwargs) # type: ignore From 94dc6936e73114ac4adc5f766d06f6c3cc9f67e8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 Mar 2021 14:14:32 -0800 Subject: [PATCH 27/39] Final update to certbot-auto (#8706) Fixes https://github.com/certbot/certbot/issues/8690. After this PR, we'll let the release script make its automated changes to certbot-auto as part of the 1.14.0 release and then never make any code changes to certbot-auto ever again! * disable upgrades on debian * update test_leauto_upgrades.sh * update changelog --- certbot/CHANGELOG.md | 2 +- letsencrypt-auto-source/letsencrypt-auto | 1 + .../letsencrypt-auto.template | 1 + tests/letstest/scripts/test_leauto_upgrades.sh | 17 ++++------------- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 7e877574f..c316ec376 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -10,7 +10,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* +* certbot-auto no longer checks for updates on any operating system. ### Fixed diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 7f1987439..b0b25759f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -800,6 +800,7 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 70b75176e..72876bb59 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -322,6 +322,7 @@ DeterminePythonVersion() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 407a865f2..3964364e1 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -153,17 +153,8 @@ if ! ./letsencrypt-auto -v --debug --version 2>&1 | grep "will no longer receive exit 1 fi -# Finally, we check if our local server received more requests. Over time, -# we'll move more and more OSes into this case until it this is the expected -# behavior on all systems. -if [ -f /etc/redhat-release ]; then - if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then - echo our local server received unexpected requests - exit 1 - fi -else - if diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then - echo our local server did not receive the requests we expected - exit 1 - fi +# Finally, we check if our local server received more requests. +if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then + echo our local server received unexpected requests + exit 1 fi From c02b2d30f2535a254020184912a94c6ed19b5026 Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Sat, 6 Mar 2021 01:53:20 +0100 Subject: [PATCH 28/39] Removed Python legacy __future__ imports (#8697) There are still some left, but the `modification_check` test fails. Some are still in `tools`, and they can probably be removed as well. `with_statement` was introduced officially in Python 2.5, so there's really old stuff in the code base. --- certbot-ci/certbot_integration_tests/assets/hook.py | 1 - .../certbot_integration_tests/certbot_tests/test_main.py | 1 - certbot-ci/certbot_integration_tests/conftest.py | 1 - certbot-ci/certbot_integration_tests/utils/acme_server.py | 1 - certbot-ci/certbot_integration_tests/utils/certbot_call.py | 1 - certbot-ci/certbot_integration_tests/utils/dns_server.py | 2 -- certbot-ci/certbot_integration_tests/utils/misc.py | 2 +- certbot-ci/windows_installer_integration_tests/conftest.py | 2 +- certbot/certbot/_internal/cli/__init__.py | 1 - certbot/certbot/_internal/cli/helpful.py | 2 +- certbot/certbot/_internal/hooks.py | 1 - certbot/certbot/_internal/log.py | 2 +- certbot/certbot/_internal/main.py | 1 - certbot/certbot/_internal/plugins/selection.py | 1 - certbot/certbot/_internal/renewal.py | 1 - certbot/certbot/_internal/reporter.py | 2 -- certbot/tests/main_test.py | 4 +--- certbot/tests/plugins/manual_test.py | 2 +- tests/letstest/multitester.py | 4 ---- tests/lock_test.py | 2 -- tools/install_and_test.py | 2 -- tools/merge_requirements.py | 2 -- tools/pip_install_editable.py | 3 --- tools/pipstrap.py | 1 - tools/readlink.py | 2 -- 25 files changed, 6 insertions(+), 38 deletions(-) diff --git a/certbot-ci/certbot_integration_tests/assets/hook.py b/certbot-ci/certbot_integration_tests/assets/hook.py index 483892a93..c11704f47 100755 --- a/certbot-ci/certbot_integration_tests/assets/hook.py +++ b/certbot-ci/certbot_integration_tests/assets/hook.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function import os import sys diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 2d3d93669..0b649acdc 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -1,5 +1,4 @@ """Module executing integration tests against certbot core.""" -from __future__ import print_function import os from os.path import exists diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index 230fb0eda..6fc77480d 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -6,7 +6,6 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ -from __future__ import print_function import contextlib import subprocess import sys diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 846e2d071..b16aa80af 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -1,6 +1,5 @@ #!/usr/bin/env python """Module to setup an ACME CA server environment able to run multiple tests in parallel""" -from __future__ import print_function import argparse import errno diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py index c9e46cdc7..b319eca4c 100755 --- a/certbot-ci/certbot_integration_tests/utils/certbot_call.py +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -1,6 +1,5 @@ #!/usr/bin/env python """Module to call certbot in test mode""" -from __future__ import absolute_import import os import subprocess diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index 62a58275e..eaa601f1b 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -1,7 +1,5 @@ #!/usr/bin/env python """Module to setup an RFC2136-capable DNS server""" -from __future__ import print_function - import os import os.path import shutil diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index e6b9f0c88..cf2e7ff09 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -311,7 +311,7 @@ def echo(keyword, path=None): if not re.match(r'^\w+$', keyword): raise ValueError('Error, keyword `{0}` is not a single keyword.' .format(keyword)) - return '{0} -c "from __future__ import print_function; print(\'{1}\')"{2}'.format( + return '{0} -c "print(\'{1}\')"{2}'.format( os.path.basename(sys.executable), keyword, ' >> "{0}"'.format(path) if path else '') diff --git a/certbot-ci/windows_installer_integration_tests/conftest.py b/certbot-ci/windows_installer_integration_tests/conftest.py index c6a89c323..8a9de057f 100644 --- a/certbot-ci/windows_installer_integration_tests/conftest.py +++ b/certbot-ci/windows_installer_integration_tests/conftest.py @@ -6,7 +6,7 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ -from __future__ import print_function + import os ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index 8d2f7c329..688efeb7a 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -1,6 +1,5 @@ """Certbot command line argument & config processing.""" # pylint: disable=too-many-lines -from __future__ import print_function import logging import logging.handlers import argparse diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index 5eaec978b..2505c24c6 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -1,5 +1,5 @@ """Certbot command line argument parser""" -from __future__ import print_function + import argparse import copy import functools diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index 5526b21c4..256fce532 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -1,5 +1,4 @@ """Facilities for implementing hooks that call shell commands.""" -from __future__ import print_function import logging diff --git a/certbot/certbot/_internal/log.py b/certbot/certbot/_internal/log.py index 90dc2cda1..d8a55e27d 100644 --- a/certbot/certbot/_internal/log.py +++ b/certbot/certbot/_internal/log.py @@ -19,7 +19,7 @@ The preferred method to display important information to the user is to use `certbot.display.util` and `certbot.display.ops`. """ -from __future__ import print_function + import functools import logging diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index f7455db96..9b2141b5a 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -1,6 +1,5 @@ """Certbot main entry point.""" # pylint: disable=too-many-lines -from __future__ import print_function import functools import logging.handlers diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index e5c311efe..bf7e505b9 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -1,5 +1,4 @@ """Decide which plugins to use for authentication & installation""" -from __future__ import print_function import logging diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index 7533c8c6b..0b99ae3d1 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -1,5 +1,4 @@ """Functionality for autorenewal and associated juggling of configurations""" -from __future__ import print_function import copy import itertools diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index f43f0e7af..3093ef8ca 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -1,6 +1,4 @@ """Collects and displays information to the user.""" -from __future__ import print_function - import collections import logging import queue diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 5ad11b120..7668a1b6a 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,8 +1,6 @@ # coding=utf-8 """Tests for certbot._internal.main.""" # pylint: disable=too-many-lines -from __future__ import print_function - import datetime from importlib import reload as reload_module import io @@ -1353,7 +1351,7 @@ class MainTest(test_util.ConfigTestCase): _, _, stdout = self._test_renewal_common( due_for_renewal=False, extra_args=None, should_renew=False, args=['renew', '--post-hook', - '{0} -c "from __future__ import print_function; print(\'hello world\');"' + '{0} -c "print(\'hello world\');"' .format(sys.executable)]) self.assertTrue('No hooks were run.' in stdout.getvalue()) diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index f3c580517..0e552e0c5 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -60,7 +60,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): def test_script_perform(self): self.config.manual_auth_hook = ( - '{0} -c "from __future__ import print_function;' + '{0} -c "' 'from certbot.compat import os;' 'print(os.environ.get(\'CERTBOT_DOMAIN\'));' 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 75dd06ad6..1907995c2 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -27,10 +27,6 @@ see: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html """ - -from __future__ import print_function -from __future__ import with_statement - import argparse import multiprocessing as mp from multiprocessing import Manager diff --git a/tests/lock_test.py b/tests/lock_test.py index 56399c874..f310b5753 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -1,6 +1,4 @@ """Tests to ensure the lock order is preserved.""" -from __future__ import print_function - import atexit import datetime import functools diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 0b47fa5f8..e7a34286c 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -5,8 +5,6 @@ # set to 1, packages are installed using pinned versions of all of our # dependencies. See pip_install.py for more information on the versions pinned # to. -from __future__ import print_function - import os import re import subprocess diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index bbcb38051..44a61249b 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -6,8 +6,6 @@ Only the simple formats SomeProject==1.2.3 or SomeProject<=1.2.3 are currently supported. """ -from __future__ import print_function - import sys diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index abfe9f214..de2a0ff57 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -6,9 +6,6 @@ # https://github.com/pyca/cryptography/blob/a02fdd60d98273ca34427235c4ca96687a12b239/.travis/downstream.d/certbot.sh#L8-L9. # We should try to remember to keep their repo updated if we make any changes # to this script which may break things for them. - -from __future__ import absolute_import - import sys import pip_install diff --git a/tools/pipstrap.py b/tools/pipstrap.py index e6b746916..df3e46003 100755 --- a/tools/pipstrap.py +++ b/tools/pipstrap.py @@ -1,6 +1,5 @@ #!/usr/bin/env python """Uses pip to upgrade Python packaging tools to pinned versions.""" -from __future__ import absolute_import import os import pip_install diff --git a/tools/readlink.py b/tools/readlink.py index 446c8ebdc..c1e0c2e61 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -6,8 +6,6 @@ useful as there are often differences in readlink on different platforms. """ -from __future__ import print_function - import os import sys From 76895457c9f5956ec97ccadc3e0d7e894db17094 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Wed, 10 Mar 2021 10:56:51 +1100 Subject: [PATCH 29/39] dns-digitalocean: use a 30 second TTL (#8693) Fixes an issue with long TTLs and caching behavior on DigitalOcean's DNS hosting service. --- .../_internal/dns_digitalocean.py | 13 +++++++++---- certbot-dns-digitalocean/setup.py | 2 +- .../tests/dns_digitalocean_test.py | 15 +++++++++------ certbot/CHANGELOG.md | 1 + tools/dev_constraints.txt | 3 ++- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index 5893df9c2..cb5012fb7 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -21,6 +21,7 @@ class Authenticator(dns_common.DNSAuthenticator): description = 'Obtain certificates using a DNS TXT record (if you are ' + \ 'using DigitalOcean for DNS).' + ttl = 30 def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) @@ -45,7 +46,8 @@ class Authenticator(dns_common.DNSAuthenticator): ) def _perform(self, domain, validation_name, validation): - self._get_digitalocean_client().add_txt_record(domain, validation_name, validation) + self._get_digitalocean_client().add_txt_record(domain, validation_name, validation, + self.ttl) def _cleanup(self, domain, validation_name, validation): self._get_digitalocean_client().del_txt_record(domain, validation_name, validation) @@ -62,13 +64,15 @@ class _DigitalOceanClient: def __init__(self, token): self.manager = digitalocean.Manager(token=token) - def add_txt_record(self, domain_name, record_name, record_content): + def add_txt_record(self, domain_name: str, record_name: str, record_content: str, + record_ttl: int): """ Add a TXT record using the supplied information. :param str domain_name: The domain to use to associate the record with. :param str record_name: The record name (typically beginning with '_acme-challenge.'). :param str record_content: The record content (typically the challenge validation). + :param int record_ttl: The record TTL. :raises certbot.errors.PluginError: if an error occurs communicating with the DigitalOcean API """ @@ -89,7 +93,8 @@ class _DigitalOceanClient: result = domain.create_new_domain_record( type='TXT', name=self._compute_record_name(domain, record_name), - data=record_content) + data=record_content, + ttl=record_ttl) # ttl kwarg is only effective starting python-digitalocean 1.15.0 record_id = result['domain_record']['id'] @@ -99,7 +104,7 @@ class _DigitalOceanClient: raise errors.PluginError('Error adding TXT record using the DigitalOcean API: {0}' .format(e)) - def del_txt_record(self, domain_name, record_name, record_content): + def del_txt_record(self, domain_name: str, record_name: str, record_content: str): """ Delete a TXT record using the supplied information. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 763961e79..106eb3cac 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -9,7 +9,7 @@ version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'python-digitalocean>=1.11', + 'python-digitalocean>=1.11', # 1.15.0 or newer is recommended for TTL support 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py index a752f52d0..84bb35b1f 100644 --- a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py @@ -40,7 +40,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def test_perform(self): self.auth.perform([self.achall]) - expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] + expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY, 30)] self.assertEqual(expected, self.mock_client.mock_calls) def test_cleanup(self): @@ -58,6 +58,7 @@ class DigitalOceanClientTest(unittest.TestCase): record_prefix = "_acme-challenge" record_name = record_prefix + "." + DOMAIN record_content = "bar" + record_ttl = 60 def setUp(self): from certbot_dns_digitalocean._internal.dns_digitalocean import _DigitalOceanClient @@ -78,25 +79,27 @@ class DigitalOceanClientTest(unittest.TestCase): self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock] - self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content) + self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content, + self.record_ttl) domain_mock.create_new_domain_record.assert_called_with(type='TXT', name=self.record_prefix, - data=self.record_content) + data=self.record_content, + ttl=self.record_ttl) def test_add_txt_record_fail_to_find_domain(self): self.manager.get_all_domains.return_value = [] self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_add_txt_record_error_finding_domain(self): self.manager.get_all_domains.side_effect = API_ERROR self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_add_txt_record_error_creating_record(self): domain_mock = mock.MagicMock() @@ -107,7 +110,7 @@ class DigitalOceanClientTest(unittest.TestCase): self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_del_txt_record(self): first_record_mock = mock.MagicMock() diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index c316ec376..ccbe6a43b 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -11,6 +11,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * certbot-auto no longer checks for updates on any operating system. +* The DigitalOcean plugin now creates TXT records for the DNS-01 challenge with a lower 30s TTL. ### Fixed diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 10308bd39..f4059a3f9 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -51,6 +51,7 @@ jedi==0.17.1 Jinja2==2.9.6 jmespath==0.9.4 josepy==1.1.0 +jsonpickle==2.0.0 jsonschema==2.6.0 lazy-object-proxy==1.4.3 logger==1.4 @@ -93,7 +94,7 @@ pytest-xdist==1.22.5 pytest-sugar==0.9.2 pytest-rerunfailures==4.2 python-dateutil==2.8.1 -python-digitalocean==1.11 +python-digitalocean==1.15.0 python-dotenv==0.14.0 pywin32==300 PyYAML==5.3.1 From 67b65bb2c0b5fee3b97ae2e94649e8ef28fcbd74 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 10 Mar 2021 01:12:32 +0100 Subject: [PATCH 30/39] Deprecate acme.typing_magic module, stop using it in certbot (#8643) * Deprecate acme.magic_typing, stop to use it in certbot * Isort * Add a changelog entry Co-authored-by: Brad Warren --- acme/acme/challenges.py | 7 ++++--- acme/acme/client.py | 8 ++++---- acme/acme/crypto_util.py | 6 +++--- acme/acme/magic_typing.py | 3 +++ acme/acme/standalone.py | 2 +- acme/tests/magic_typing_test.py | 11 +++++++---- .../certbot_apache/_internal/apache_util.py | 1 - .../certbot_apache/_internal/assertions.py | 1 - .../certbot_apache/_internal/augeasparser.py | 4 ++-- .../certbot_apache/_internal/configurator.py | 13 +++++++------ .../certbot_apache/_internal/dualparser.py | 2 +- .../certbot_apache/_internal/http_01.py | 6 +++--- .../certbot_apache/_internal/interfaces.py | 1 - certbot-apache/certbot_apache/_internal/obj.py | 2 +- .../certbot_apache/_internal/override_centos.py | 2 +- certbot-apache/certbot_apache/_internal/parser.py | 5 +++-- .../nginx_tests/test_main.py | 2 +- .../rfc2136_tests/context.py | 2 +- .../utils/acme_server.py | 4 ++-- .../certbot_integration_tests/utils/misc.py | 4 ++-- .../utils/pebble_artifacts.py | 3 ++- .../snap_integration_tests/dns_tests/test_main.py | 5 +++-- .../test_main.py | 4 ++-- .../configurators/nginx/common.py | 2 +- .../certbot_compatibility_test/test_driver.py | 4 ++-- .../_internal/dns_cloudflare.py | 7 +++---- .../certbot_dns_route53/_internal/dns_route53.py | 6 +++--- .../certbot_nginx/_internal/configurator.py | 12 ++++++------ .../certbot_nginx/_internal/constants.py | 5 ++--- certbot-nginx/certbot_nginx/_internal/http_01.py | 4 ++-- .../certbot_nginx/_internal/nginxparser.py | 3 ++- certbot-nginx/certbot_nginx/_internal/parser.py | 12 ++++++------ .../certbot_nginx/_internal/parser_obj.py | 2 +- certbot/CHANGELOG.md | 2 ++ certbot/certbot/_internal/account.py | 2 +- certbot/certbot/_internal/auth_handler.py | 6 +++--- certbot/certbot/_internal/cert_manager.py | 2 +- certbot/certbot/_internal/cli/__init__.py | 4 ++-- certbot/certbot/_internal/cli/cli_utils.py | 4 ++-- certbot/certbot/_internal/cli/helpful.py | 4 ++-- certbot/certbot/_internal/client.py | 4 ++-- certbot/certbot/_internal/eff.py | 5 ++--- certbot/certbot/_internal/error_handler.py | 10 +++++----- certbot/certbot/_internal/hooks.py | 4 ++-- certbot/certbot/_internal/lock.py | 2 +- certbot/certbot/_internal/main.py | 6 +++++- certbot/certbot/_internal/plugins/disco.py | 2 +- certbot/certbot/_internal/plugins/manual.py | 3 ++- certbot/certbot/_internal/plugins/standalone.py | 10 +++++----- certbot/certbot/_internal/plugins/webroot.py | 8 ++++---- certbot/certbot/_internal/renewal.py | 9 +++++---- certbot/certbot/_internal/snap_config.py | 2 +- certbot/certbot/compat/filesystem.py | 3 +-- certbot/certbot/compat/misc.py | 4 ++-- certbot/certbot/crypto_util.py | 15 +++++++++------ certbot/certbot/display/util.py | 2 +- certbot/certbot/ocsp.py | 4 ++-- certbot/certbot/plugins/common.py | 2 +- certbot/certbot/plugins/dns_common_lexicon.py | 6 +++--- certbot/certbot/plugins/enhancements.py | 6 +++--- certbot/certbot/plugins/storage.py | 4 ++-- certbot/certbot/util.py | 8 ++++---- tests/letstest/multitester.py | 1 - tools/pipstrap.py | 1 - tools/snap/build_remote.py | 10 ++++++++-- 65 files changed, 165 insertions(+), 145 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 41a2aa258..58b457e0d 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -8,14 +8,15 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose -import requests -from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from OpenSSL import crypto +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 +import requests from acme import crypto_util from acme import errors from acme import fields -from acme.mixins import ResourceMixin, TypeMixin +from acme.mixins import ResourceMixin +from acme.mixins import TypeMixin logger = logging.getLogger(__name__) diff --git a/acme/acme/client.py b/acme/acme/client.py index 33e515124..b1b2089cf 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -8,6 +8,10 @@ import http.client as http_client import logging import re import time +from typing import Dict +from typing import List +from typing import Set +from typing import Text import josepy as jose import OpenSSL @@ -20,10 +24,6 @@ from acme import crypto_util from acme import errors from acme import jws from acme import messages -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Text from acme.mixins import VersionedLEACMEMixin logger = logging.getLogger(__name__) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index a14737053..04faf6912 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -5,15 +5,15 @@ import logging import os import re import socket +from typing import Callable +from typing import Tuple +from typing import Union import josepy as jose from OpenSSL import crypto from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from acme import errors -from acme.magic_typing import Callable -from acme.magic_typing import Tuple -from acme.magic_typing import Union logger = logging.getLogger(__name__) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 91308fef6..8190fa552 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -4,9 +4,12 @@ This was useful when this code supported Python 2 and typing wasn't always available. This code is being kept for now for backwards compatibility. """ +import warnings from typing import * # pylint: disable=wildcard-import, unused-wildcard-import from typing import Collection, IO # type: ignore +warnings.warn("acme.magic_typing is deprecated and will be removed in a future release.", + DeprecationWarning) class TypingClass: """Ignore import errors by getting anything""" diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index f5bc548b6..123899470 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -7,10 +7,10 @@ import logging import socket import socketserver import threading +from typing import List from acme import challenges from acme import crypto_util -from acme.magic_typing import List logger = logging.getLogger(__name__) diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 257164d77..d470337bd 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -1,6 +1,7 @@ """Tests for acme.magic_typing.""" import sys import unittest +import warnings from unittest import mock @@ -9,15 +10,17 @@ class MagicTypingTest(unittest.TestCase): def test_import_success(self): try: import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover typing_class_mock = mock.MagicMock() text_mock = mock.MagicMock() typing_class_mock.Text = text_mock sys.modules['typing'] = typing_class_mock if 'acme.magic_typing' in sys.modules: - del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text + del sys.modules['acme.magic_typing'] # pragma: no cover + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + from acme.magic_typing import Text self.assertEqual(Text, text_mock) del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing diff --git a/certbot-apache/certbot_apache/_internal/apache_util.py b/certbot-apache/certbot_apache/_internal/apache_util.py index 93612f424..9b9855a45 100644 --- a/certbot-apache/certbot_apache/_internal/apache_util.py +++ b/certbot-apache/certbot_apache/_internal/apache_util.py @@ -9,7 +9,6 @@ import pkg_resources from certbot import errors from certbot import util - from certbot.compat import os logger = logging.getLogger(__name__) diff --git a/certbot-apache/certbot_apache/_internal/assertions.py b/certbot-apache/certbot_apache/_internal/assertions.py index 2b2ce4f50..53603c526 100644 --- a/certbot-apache/certbot_apache/_internal/assertions.py +++ b/certbot-apache/certbot_apache/_internal/assertions.py @@ -3,7 +3,6 @@ import fnmatch from certbot_apache._internal import interfaces - PASS = "CERTBOT_PASS_ASSERT" diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index 3b2ce40d8..461b5cb7b 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -64,10 +64,10 @@ Translates over to: "/files/etc/apache2/apache2.conf/bLoCk[1]", ] """ -from acme.magic_typing import Set +from typing import Set + from certbot import errors from certbot.compat import os - from certbot_apache._internal import apache_util from certbot_apache._internal import assertions from certbot_apache._internal import interfaces diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 16def1998..ecd5c46de 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -1,13 +1,18 @@ """Apache Configurator.""" # pylint: disable=too-many-lines from collections import defaultdict -from distutils.version import LooseVersion import copy +from distutils.version import LooseVersion import fnmatch import logging import re import socket import time +from typing import DefaultDict +from typing import Dict +from typing import List +from typing import Set +from typing import Union import zope.component import zope.interface @@ -18,11 +23,6 @@ except ImportError: # pragma: no cover HAS_APACHECONFIG = False from acme import challenges -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Union from certbot import errors from certbot import interfaces from certbot import util @@ -41,6 +41,7 @@ from certbot_apache._internal import http_01 from certbot_apache._internal import obj from certbot_apache._internal import parser + logger = logging.getLogger(__name__) diff --git a/certbot-apache/certbot_apache/_internal/dualparser.py b/certbot-apache/certbot_apache/_internal/dualparser.py index 1ba23e92f..c89ff95be 100644 --- a/certbot-apache/certbot_apache/_internal/dualparser.py +++ b/certbot-apache/certbot_apache/_internal/dualparser.py @@ -1,7 +1,7 @@ """ Dual ParserNode implementation """ +from certbot_apache._internal import apacheparser from certbot_apache._internal import assertions from certbot_apache._internal import augeasparser -from certbot_apache._internal import apacheparser class DualNodeBase: diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 5ef44fa2e..4bd935d2d 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -1,9 +1,9 @@ """A class that performs HTTP-01 challenges for Apache""" -import logging import errno +import logging +from typing import List +from typing import Set -from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot.compat import filesystem from certbot.compat import os diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 77daa5b56..38c21b785 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -102,7 +102,6 @@ For this reason the internal representation of data should not ignore the case. import abc - class ParserNode(object, metaclass=abc.ABCMeta): """ ParserNode is the basic building block of the tree of such nodes, diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index e2fe48cf8..5e4680e1b 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -1,7 +1,7 @@ """Module contains classes used by the Apache Configurator.""" import re +from typing import Set -from acme.magic_typing import Set from certbot.plugins import common diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 9b2ee54c9..4cde65d57 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -1,9 +1,9 @@ """ Distribution specific override class for CentOS family (RHEL, Fedora) """ import logging +from typing import List import zope.interface -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot import util diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index a1ea02b18..eb8f267f9 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -3,9 +3,10 @@ import copy import fnmatch import logging import re +from typing import Dict +from typing import List + -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py index 8a2d48a50..4c746e1ee 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -1,8 +1,8 @@ """Module executing integration tests against certbot with nginx plugin.""" import os import ssl - from typing import List + import pytest from certbot_integration_tests.nginx_tests import context as nginx_context diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py index bdedee1fe..c8024b21d 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -1,7 +1,7 @@ """Module to handle the context of RFC2136 integration tests.""" -import tempfile from contextlib import contextmanager +import tempfile from pkg_resources import resource_filename from pytest import skip diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index b16aa80af..706c3ebae 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -11,14 +11,14 @@ import subprocess import sys import tempfile import time - from typing import List + import requests +# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils import misc from certbot_integration_tests.utils import pebble_artifacts from certbot_integration_tests.utils import proxy -# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils.constants import * diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index cf2e7ff09..2fac494f2 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -26,8 +26,8 @@ from OpenSSL import crypto import pkg_resources import requests -from certbot_integration_tests.utils.constants import \ - PEBBLE_ALTERNATE_ROOTS, PEBBLE_MANAGEMENT_URL +from certbot_integration_tests.utils.constants import PEBBLE_ALTERNATE_ROOTS +from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL RSA_KEY_TYPE = 'rsa' ECDSA_KEY_TYPE = 'ecdsa' diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index cd62e1a7f..918a5fd04 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -7,7 +7,8 @@ import stat import pkg_resources import requests -from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT, MOCK_OCSP_SERVER_PORT +from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT +from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT PEBBLE_VERSION = 'v2.3.0' ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') diff --git a/certbot-ci/snap_integration_tests/dns_tests/test_main.py b/certbot-ci/snap_integration_tests/dns_tests/test_main.py index 016355334..721352c04 100644 --- a/certbot-ci/snap_integration_tests/dns_tests/test_main.py +++ b/certbot-ci/snap_integration_tests/dns_tests/test_main.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -import pytest -import subprocess import glob import os import re +import subprocess + +import pytest @pytest.fixture(autouse=True, scope="module") diff --git a/certbot-ci/windows_installer_integration_tests/test_main.py b/certbot-ci/windows_installer_integration_tests/test_main.py index c8c347aa8..f699b736a 100644 --- a/certbot-ci/windows_installer_integration_tests/test_main.py +++ b/certbot-ci/windows_installer_integration_tests/test_main.py @@ -1,8 +1,8 @@ import os +import re +import subprocess import time import unittest -import subprocess -import re @unittest.skipIf(os.name != 'nt', reason='Windows installer tests must be run on Windows.') diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py index 7cb4e9722..58b72c205 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py @@ -2,10 +2,10 @@ import os import shutil import subprocess +from typing import Set import zope.interface -from acme.magic_typing import Set from certbot._internal import configuration from certbot_compatibility_test import errors from certbot_compatibility_test import interfaces diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index f11b9fdf8..fe634ba9f 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -8,6 +8,8 @@ import shutil import sys import tempfile import time +from typing import List +from typing import Tuple import OpenSSL from urllib3.util import connection @@ -15,8 +17,6 @@ from urllib3.util import connection from acme import challenges from acme import crypto_util from acme import messages -from acme.magic_typing import List -from acme.magic_typing import Tuple from certbot import achallenges from certbot import errors as le_errors from certbot.tests import acme_util diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index c896a6e3a..583b41410 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -1,13 +1,12 @@ """DNS Authenticator for Cloudflare.""" import logging +from typing import Any +from typing import Dict +from typing import List import CloudFlare import zope.interface -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import List - from certbot import errors from certbot import interfaces from certbot.plugins import dns_common diff --git a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py index 6250d2274..4837e2036 100644 --- a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py @@ -2,15 +2,15 @@ import collections import logging import time +from typing import DefaultDict +from typing import Dict +from typing import List import boto3 from botocore.exceptions import ClientError from botocore.exceptions import NoCredentialsError import zope.interface -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot.plugins import dns_common diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 8a3b8078f..33baf076a 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -7,6 +7,12 @@ import socket import subprocess import tempfile import time +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Text +from typing import Tuple import OpenSSL import pkg_resources @@ -14,12 +20,6 @@ import zope.interface from acme import challenges from acme import crypto_util as acme_crypto_util -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Optional -from acme.magic_typing import Set -from acme.magic_typing import Text -from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import interfaces diff --git a/certbot-nginx/certbot_nginx/_internal/constants.py b/certbot-nginx/certbot_nginx/_internal/constants.py index 1f058e7ef..7b4111577 100644 --- a/certbot-nginx/certbot_nginx/_internal/constants.py +++ b/certbot-nginx/certbot_nginx/_internal/constants.py @@ -1,8 +1,7 @@ """nginx plugin constants.""" import platform - -from acme.magic_typing import Any -from acme.magic_typing import Dict +from typing import Any +from typing import Dict FREEBSD_DARWIN_SERVER_ROOT = "/usr/local/etc/nginx" LINUX_SERVER_ROOT = "/etc/nginx" diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index 896760fc4..ec610f15b 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -2,10 +2,10 @@ import io import logging +from typing import List +from typing import Optional from acme import challenges -from acme.magic_typing import List -from acme.magic_typing import Optional from certbot import achallenges from certbot import errors from certbot.compat import os diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index a51302fae..1aee1f329 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -2,6 +2,8 @@ # Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed) import copy import logging +from typing import Any # pylint: disable=unused-import +from typing import IO # pylint: disable=unused-import from pyparsing import Combine from pyparsing import Forward @@ -15,7 +17,6 @@ from pyparsing import restOfLine from pyparsing import stringEnd from pyparsing import White from pyparsing import ZeroOrMore -from acme.magic_typing import IO, Any # pylint: disable=unused-import logger = logging.getLogger(__name__) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index db9530104..ed7e211d5 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -5,15 +5,15 @@ import glob import io import logging import re +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Tuple +from typing import Union import pyparsing -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Optional -from acme.magic_typing import Set -from acme.magic_typing import Tuple -from acme.magic_typing import Union from certbot import errors from certbot.compat import os from certbot_nginx._internal import nginxparser diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index e55d48dc4..9efd6651a 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -3,9 +3,9 @@ raw lists of tokens from pyparsing. """ import abc import logging +from typing import List -from acme.magic_typing import List from certbot import errors logger = logging.getLogger(__name__) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index ccbe6a43b..62bceb74d 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -11,6 +11,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * certbot-auto no longer checks for updates on any operating system. +* The module `acme.magic_typing` is deprecated and will be removed in a future release. + Please use the built-in module `typing` instead. * The DigitalOcean plugin now creates TXT records for the DNS-01 challenge with a lower 30s TTL. ### Fixed diff --git a/certbot/certbot/_internal/account.py b/certbot/certbot/_internal/account.py index b2d50297e..07ae95e9d 100644 --- a/certbot/certbot/_internal/account.py +++ b/certbot/certbot/_internal/account.py @@ -18,8 +18,8 @@ from certbot import errors from certbot import interfaces from certbot import util from certbot._internal import constants -from certbot.compat import os from certbot.compat import filesystem +from certbot.compat import os logger = logging.getLogger(__name__) diff --git a/certbot/certbot/_internal/auth_handler.py b/certbot/certbot/_internal/auth_handler.py index 17bf75225..2801e3c2f 100644 --- a/certbot/certbot/_internal/auth_handler.py +++ b/certbot/certbot/_internal/auth_handler.py @@ -2,15 +2,15 @@ import datetime import logging import time +from typing import Dict +from typing import List +from typing import Tuple import zope.component from acme import challenges from acme import errors as acme_errors from acme import messages -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Tuple from certbot import achallenges from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index ee2bd6254..2ac9fc538 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -3,11 +3,11 @@ import datetime import logging import re import traceback +from typing import List import pytz import zope.component -from acme.magic_typing import List from certbot import crypto_util from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index 688efeb7a..30f444e5b 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -4,11 +4,11 @@ import logging import logging.handlers import argparse import sys +from typing import Optional + import certbot._internal.plugins.selection as plugin_selection from certbot._internal.plugins import disco as plugins_disco -from acme.magic_typing import Optional - # pylint: disable=ungrouped-imports import certbot from certbot._internal import constants diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index 5bdbbe02c..f1f9d0f3c 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -5,11 +5,11 @@ import copy import zope.interface.interface # pylint: disable=unused-import from acme import challenges +from certbot import errors from certbot import interfaces from certbot import util -from certbot import errors -from certbot.compat import os from certbot._internal import constants +from certbot.compat import os class _Default: diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index 2505c24c6..cbac4e232 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -5,6 +5,8 @@ import copy import functools import glob import sys +from typing import Any +from typing import Dict import configargparse import zope.component @@ -12,8 +14,6 @@ import zope.interface from zope.interface import interfaces as zope_interfaces -from acme.magic_typing import Any, Dict - from certbot import crypto_util from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index f2fc06937..b9137e249 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -2,6 +2,8 @@ import datetime import logging import platform +from typing import List +from typing import Optional from cryptography.hazmat.backends import default_backend # See https://github.com/pyca/cryptography/issues/4275 @@ -14,8 +16,6 @@ from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors from acme import messages -from acme.magic_typing import List -from acme.magic_typing import Optional import certbot from certbot import crypto_util from certbot import errors diff --git a/certbot/certbot/_internal/eff.py b/certbot/certbot/_internal/eff.py index 5fbbd302a..6212c24ee 100644 --- a/certbot/certbot/_internal/eff.py +++ b/certbot/certbot/_internal/eff.py @@ -1,16 +1,15 @@ """Subscribes users to the EFF newsletter.""" import logging +from typing import Optional # pylint: disable=unused-import import requests import zope.component -from acme.magic_typing import Optional # pylint: disable=unused-import - from certbot import interfaces -from certbot.display import util as display_util from certbot._internal import constants from certbot._internal.account import Account # pylint: disable=unused-import from certbot._internal.account import AccountFileStorage +from certbot.display import util as display_util from certbot.interfaces import IConfig # pylint: disable=unused-import logger = logging.getLogger(__name__) diff --git a/certbot/certbot/_internal/error_handler.py b/certbot/certbot/_internal/error_handler.py index 05af9d837..cb8301980 100644 --- a/certbot/certbot/_internal/error_handler.py +++ b/certbot/certbot/_internal/error_handler.py @@ -3,12 +3,12 @@ import functools import logging import signal import traceback +from typing import Any +from typing import Callable +from typing import Dict +from typing import List +from typing import Union -from acme.magic_typing import Any -from acme.magic_typing import Callable -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Union from certbot import errors from certbot.compat import os diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index 256fce532..b695d9930 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -1,9 +1,9 @@ """Facilities for implementing hooks that call shell commands.""" import logging +from typing import List +from typing import Set -from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot import util from certbot.compat import filesystem diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index 32142fe44..f878946cb 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -1,8 +1,8 @@ """Implements file locks compatible with Linux and Windows for locking files and directories.""" import errno import logging +from typing import Optional -from acme.magic_typing import Optional from certbot import errors from certbot.compat import filesystem from certbot.compat import os diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 9b2141b5a..ba8d94bca 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -4,13 +4,17 @@ import functools import logging.handlers import sys +from typing import Iterable # pylint: disable=unused-import +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union import configobj import josepy as jose import zope.component from acme import errors as acme_errors -from acme.magic_typing import Union, Iterable, Optional, List, Tuple # pylint: disable=unused-import import certbot from certbot import crypto_util from certbot import errors diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index 1e1e00982..27b2605e0 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -2,13 +2,13 @@ import itertools import logging import sys +from typing import Dict from collections.abc import Mapping import pkg_resources import zope.interface import zope.interface.verify -from acme.magic_typing import Dict from certbot import errors from certbot import interfaces from certbot._internal import constants diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index a2e4bb28e..c28e333b9 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -1,9 +1,10 @@ """Manual authenticator plugin""" +from typing import Dict + import zope.component import zope.interface from acme import challenges -from acme.magic_typing import Dict from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index d5d9fd2ec..f511c60e9 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -4,17 +4,17 @@ import logging import socket # https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi from socket import errno as socket_errors # type: ignore +from typing import DefaultDict +from typing import Dict +from typing import Set +from typing import Tuple +from typing import TYPE_CHECKING import OpenSSL # pylint: disable=unused-import import zope.interface from acme import challenges from acme import standalone as acme_standalone -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import Set -from acme.magic_typing import Tuple -from acme.magic_typing import TYPE_CHECKING from certbot import achallenges from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index 8789db604..730241066 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -3,15 +3,15 @@ import argparse import collections import json import logging +from typing import DefaultDict +from typing import Dict +from typing import List +from typing import Set import zope.component import zope.interface from acme import challenges -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index 0b99ae3d1..a09edfa1e 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -7,17 +7,17 @@ import random import sys import time import traceback +from typing import List +from typing import Optional # pylint: disable=unused-import from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import ec, rsa +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.serialization import load_pem_private_key import OpenSSL import zope.component -from acme.magic_typing import List -from acme.magic_typing import Optional # pylint: disable=unused-import from certbot import crypto_util -from certbot.display import util as display_util from certbot import errors from certbot import interfaces from certbot import util @@ -29,6 +29,7 @@ from certbot._internal import storage from certbot._internal import updater from certbot._internal.plugins import disco as plugins_disco from certbot.compat import os +from certbot.display import util as display_util logger = logging.getLogger(__name__) diff --git a/certbot/certbot/_internal/snap_config.py b/certbot/certbot/_internal/snap_config.py index a90740787..f2fe035db 100644 --- a/certbot/certbot/_internal/snap_config.py +++ b/certbot/certbot/_internal/snap_config.py @@ -1,13 +1,13 @@ """Module configuring Certbot in a snap environment""" import logging import socket +from typing import List from requests import Session from requests.adapters import HTTPAdapter from requests.exceptions import HTTPError from requests.exceptions import RequestException -from acme.magic_typing import List from certbot.compat import os from certbot.errors import Error diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index 0152685e9..aa6fe9134 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -5,8 +5,7 @@ import errno import os # pylint: disable=os-module-forbidden import stat import sys - -from acme.magic_typing import List +from typing import List try: import ntsecuritycon diff --git a/certbot/certbot/compat/misc.py b/certbot/certbot/compat/misc.py index f4ea4a5cc..297df80fc 100644 --- a/certbot/certbot/compat/misc.py +++ b/certbot/certbot/compat/misc.py @@ -8,12 +8,12 @@ import logging import select import subprocess import sys +from typing import Optional +from typing import Tuple from certbot import errors from certbot.compat import os -from acme.magic_typing import Tuple, Optional - try: from win32com.shell import shell as shellwin32 POSIX_MODE = False diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index f67d95d97..5bc9251a4 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -6,26 +6,29 @@ """ import hashlib import logging +import re +from typing import IO # pylint: disable=unused-import import warnings -import re # See https://github.com/pyca/cryptography/issues/4275 from cryptography import x509 # type: ignore -from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm +from cryptography.exceptions import InvalidSignature +from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import ec -from cryptography.hazmat.primitives.asymmetric.ec import ECDSA, EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey -from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat +from cryptography.hazmat.primitives.serialization import Encoding +from cryptography.hazmat.primitives.serialization import NoEncryption +from cryptography.hazmat.primitives.serialization import PrivateFormat from OpenSSL import crypto from OpenSSL import SSL # type: ignore - import pyrfc3339 import zope.component from acme import crypto_util as acme_crypto_util -from acme.magic_typing import IO # pylint: disable=unused-import from certbot import errors from certbot import interfaces from certbot import util diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index f26ec468f..1e85ae808 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -12,11 +12,11 @@ Other messages can use the `logging` module. See `log.py`. import logging import sys import textwrap +from typing import List import zope.interface import zope.component -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot._internal import constants diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index dcbf0381a..9a18b32f2 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -5,6 +5,8 @@ import logging import re from subprocess import PIPE from subprocess import Popen +from typing import Optional +from typing import Tuple from cryptography import x509 from cryptography.exceptions import InvalidSignature @@ -16,8 +18,6 @@ from cryptography.hazmat.primitives import serialization import pytz import requests -from acme.magic_typing import Optional -from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import util diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index 489c75a3d..5e1c29669 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -3,12 +3,12 @@ import logging import re import shutil import tempfile +from typing import List from josepy import util as jose_util import pkg_resources import zope.interface -from acme.magic_typing import List from certbot import achallenges # pylint: disable=unused-import from certbot import crypto_util from certbot import errors diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index a29509b79..5fec5e0d6 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -1,12 +1,12 @@ """Common code for DNS Authenticator Plugins built on Lexicon.""" import logging +from typing import Any +from typing import Dict +from typing import Union from requests.exceptions import HTTPError from requests.exceptions import RequestException -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import Union from certbot import errors from certbot.plugins import dns_common diff --git a/certbot/certbot/plugins/enhancements.py b/certbot/certbot/plugins/enhancements.py index e674c32a2..3c3db7e71 100644 --- a/certbot/certbot/plugins/enhancements.py +++ b/certbot/certbot/plugins/enhancements.py @@ -1,9 +1,9 @@ """New interface style Certbot enhancements""" import abc +from typing import Any +from typing import Dict +from typing import List -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot._internal import constants ENHANCEMENTS = ["redirect", "ensure-http-header", "ocsp-stapling"] diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index abef534f9..b5dd6a502 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -1,9 +1,9 @@ """Plugin storage class.""" import json import logging +from typing import Any +from typing import Dict -from acme.magic_typing import Any -from acme.magic_typing import Dict from certbot import errors from certbot.compat import filesystem from certbot.compat import os diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index ffd3a65f2..0643d5ba7 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -12,13 +12,13 @@ import re import socket import subprocess import sys +from typing import Dict +from typing import Text +from typing import Tuple +from typing import Union import configargparse -from acme.magic_typing import Dict -from acme.magic_typing import Text -from acme.magic_typing import Tuple -from acme.magic_typing import Union from certbot import errors from certbot._internal import constants from certbot._internal import lock diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 1907995c2..f079f893f 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -45,7 +45,6 @@ import yaml from fabric import Config from fabric import Connection - # Command line parser #------------------------------------------------------------------------------- parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.') diff --git a/tools/pipstrap.py b/tools/pipstrap.py index df3e46003..2b2e3dcbb 100755 --- a/tools/pipstrap.py +++ b/tools/pipstrap.py @@ -4,7 +4,6 @@ import os import pip_install - _REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_constraints.txt") diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index e6a44240f..e690d2916 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -2,12 +2,18 @@ import argparse import datetime import glob +from multiprocessing import Manager +from multiprocessing import Pool +from multiprocessing import Process +from os.path import basename +from os.path import dirname +from os.path import exists +from os.path import join +from os.path import realpath import re import subprocess import sys import time -from multiprocessing import Pool, Process, Manager -from os.path import join, realpath, dirname, basename, exists CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] From f2d8c81e9b8165a37c27641a49b447e55921147e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Mar 2021 16:53:44 -0800 Subject: [PATCH 31/39] remove reference to acme.magic_typing from docs (#8709) --- certbot/docs/contributing.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index def2c7fcd..13ac5ed68 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -478,13 +478,6 @@ to start contributing to Certbot. To run mypy on Certbot, use ``tox -e mypy`` on a machine that has Python 3 installed. -Note that instead of just importing ``typing``, due to packaging issues, in Certbot we import from -``acme.magic_typing`` and have to add some comments for pylint like this: - -.. code-block:: python - - from acme.magic_typing import Dict - Also note that OpenSSL, which we rely on, has type definitions for crypto but not SSL. We use both. Those imports should look like this: From dd6f2f565e8183b69ff3abf5d965d0d1575b2e5e Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 10 Mar 2021 20:51:27 +0100 Subject: [PATCH 32/39] Convert Python 2 type hints to Python 3 types annotations (#8640) Fixes #8427 This PR converts the Python 2 types hints into Python 3 types annotations. I have used the project https://github.com/ilevkivskyi/com2ann which has been designed for that specific purpose and did that very well. The only remaining things to do were to fix broken type hints that became wrong code after migration, and to fix lines too long with the new syntax. * Raw execution of com2ann * Fixing broken type annotations * Cleanup imports --- acme/acme/challenges.py | 4 +- acme/acme/client.py | 11 +- acme/acme/crypto_util.py | 4 +- acme/acme/messages.py | 6 +- acme/acme/standalone.py | 4 +- acme/tests/client_test.py | 2 +- acme/tests/crypto_util_test.py | 2 +- acme/tests/messages_test.py | 3 +- acme/tests/standalone_test.py | 4 +- .../certbot_apache/_internal/augeasparser.py | 2 +- .../certbot_apache/_internal/configurator.py | 33 +++--- .../certbot_apache/_internal/http_01.py | 4 +- .../certbot_apache/_internal/obj.py | 4 +- .../_internal/override_centos.py | 6 +- .../certbot_apache/_internal/parser.py | 11 +- certbot-apache/tests/augeasnode_test.py | 2 +- certbot-apache/tests/http_01_test.py | 2 +- .../nginx_tests/test_main.py | 4 +- .../utils/acme_server.py | 2 +- .../utils/dns_server.py | 5 +- .../configurators/nginx/common.py | 2 +- .../certbot_compatibility_test/test_driver.py | 2 +- .../_internal/dns_cloudflare.py | 2 +- .../_internal/dns_route53.py | 2 +- .../certbot_nginx/_internal/configurator.py | 8 +- .../certbot_nginx/_internal/constants.py | 4 +- .../certbot_nginx/_internal/http_01.py | 2 +- .../certbot_nginx/_internal/nginxparser.py | 102 +++++++++--------- .../certbot_nginx/_internal/parser.py | 15 ++- .../certbot_nginx/_internal/parser_obj.py | 9 +- certbot-nginx/tests/parser_test.py | 2 +- certbot/certbot/_internal/account.py | 18 ++-- certbot/certbot/_internal/auth_handler.py | 7 +- certbot/certbot/_internal/cert_manager.py | 4 +- certbot/certbot/_internal/cli/__init__.py | 2 +- certbot/certbot/_internal/cli/cli_utils.py | 2 +- certbot/certbot/_internal/cli/helpful.py | 4 +- certbot/certbot/_internal/client.py | 6 +- certbot/certbot/_internal/eff.py | 24 ++--- certbot/certbot/_internal/error_handler.py | 9 +- certbot/certbot/_internal/hooks.py | 4 +- certbot/certbot/_internal/lock.py | 71 +++++------- certbot/certbot/_internal/main.py | 37 +++---- certbot/certbot/_internal/plugins/disco.py | 4 +- certbot/certbot/_internal/plugins/manual.py | 5 +- .../certbot/_internal/plugins/standalone.py | 11 +- certbot/certbot/_internal/plugins/webroot.py | 11 +- certbot/certbot/_internal/renewal.py | 25 +++-- certbot/certbot/_internal/reporter.py | 2 +- certbot/certbot/_internal/snap_config.py | 3 +- certbot/certbot/compat/filesystem.py | 58 ++++------ certbot/certbot/compat/misc.py | 15 +-- certbot/certbot/crypto_util.py | 15 ++- certbot/certbot/display/util.py | 8 +- certbot/certbot/ocsp.py | 16 ++- certbot/certbot/plugins/common.py | 6 +- certbot/certbot/plugins/dns_common_lexicon.py | 7 +- certbot/certbot/plugins/enhancements.py | 4 +- certbot/certbot/plugins/storage.py | 2 +- certbot/certbot/util.py | 8 +- certbot/tests/display/completer_test.py | 2 +- certbot/tests/error_handler_test.py | 2 +- certbot/tests/hook_test.py | 2 +- certbot/tests/log_test.py | 2 +- certbot/tests/main_test.py | 8 +- certbot/tests/plugins/disco_test.py | 2 +- certbot/tests/plugins/selection_test.py | 2 +- certbot/tests/plugins/standalone_test.py | 5 +- 68 files changed, 307 insertions(+), 371 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 58b457e0d..82f7ae1b8 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) class Challenge(jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge.""" - TYPES = {} # type: dict + TYPES: dict = {} @classmethod def from_json(cls, jobj): @@ -38,7 +38,7 @@ class Challenge(jose.TypedJSONObjectWithFields): class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge response.""" - TYPES = {} # type: dict + TYPES: dict = {} resource_type = 'challenge' resource = fields.Resource(resource_type) diff --git a/acme/acme/client.py b/acme/acme/client.py index b1b2089cf..f5aa1ff9e 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -112,8 +112,9 @@ class ClientBase: """ return self.update_registration(regr, update={'status': 'deactivated'}) - def deactivate_authorization(self, authzr): - # type: (messages.AuthorizationResource) -> messages.AuthorizationResource + def deactivate_authorization(self, + authzr: messages.AuthorizationResource + ) -> messages.AuthorizationResource: """Deactivate authorization. :param messages.AuthorizationResource authzr: The Authorization resource @@ -423,7 +424,7 @@ class Client(ClientBase): """ assert max_attempts > 0 - attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int] + attempts: Dict[messages.AuthorizationResource, int] = collections.defaultdict(int) exhausted = set() # priority queue with datetime.datetime (based on Retry-After) as key, @@ -536,7 +537,7 @@ class Client(ClientBase): :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509` """ - chain = [] # type: List[jose.ComparableX509] + chain: List[jose.ComparableX509] = [] uri = certr.cert_chain_uri while uri is not None and len(chain) < max_length: response, cert = self._get_cert(uri) @@ -968,7 +969,7 @@ class ClientNetwork: self.account = account self.alg = alg self.verify_ssl = verify_ssl - self._nonces = set() # type: Set[Text] + self._nonces: Set[Text] = set() self.user_agent = user_agent self.session = requests.Session() self._default_timeout = timeout diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 04faf6912..749478bf5 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -168,7 +168,7 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu source_address[1] ) if any(source_address) else "" ) - socket_tuple = (host, port) # type: Tuple[str, int] + socket_tuple: Tuple[str, int] = (host, port) sock = socket.create_connection(socket_tuple, **socket_kwargs) # type: ignore except socket.error as error: raise errors.Error(error) @@ -256,7 +256,7 @@ def _pyopenssl_cert_or_req_san(cert_or_req): if isinstance(cert_or_req, crypto.X509): # pylint: disable=line-too-long - func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] + func: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] = crypto.dump_certificate else: func = crypto.dump_certificate_request text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 0d73037ae..5e7e22c34 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -153,7 +153,7 @@ class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore class Status(_Constant): """ACME "status" field.""" - POSSIBLE_NAMES = {} # type: dict + POSSIBLE_NAMES: dict = {} STATUS_UNKNOWN = Status('unknown') STATUS_PENDING = Status('pending') STATUS_PROCESSING = Status('processing') @@ -166,7 +166,7 @@ STATUS_DEACTIVATED = Status('deactivated') class IdentifierType(_Constant): """ACME identifier type.""" - POSSIBLE_NAMES = {} # type: dict + POSSIBLE_NAMES: dict = {} IDENTIFIER_FQDN = IdentifierType('dns') # IdentifierDNS in Boulder @@ -184,7 +184,7 @@ class Identifier(jose.JSONObjectWithFields): class Directory(jose.JSONDeSerializable): """Directory.""" - _REGISTERED_TYPES = {} # type: dict + _REGISTERED_TYPES: dict = {} class Meta(jose.JSONObjectWithFields): """Directory Meta.""" diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 123899470..0b5a8b5c7 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -63,8 +63,8 @@ class BaseDualNetworkedServers: def __init__(self, ServerClass, server_address, *remaining_args, **kwargs): port = server_address[1] - self.threads = [] # type: List[threading.Thread] - self.servers = [] # type: List[ACMEServerMixin] + self.threads: List[threading.Thread] = [] + self.servers: List[ACMEServerMixin] = [] # Must try True first. # Ubuntu, for example, will fail to bind to IPv4 if we've already bound diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 6f9aecda2..14247335c 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -61,7 +61,7 @@ class ClientTestBase(unittest.TestCase): self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212') reg = messages.Registration( contact=self.contact, key=KEY.public_key()) - the_arg = dict(reg) # type: Dict + the_arg: Dict = dict(reg) self.new_reg = messages.NewRegistration(**the_arg) self.regr = messages.RegistrationResource( body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index f271ad37d..8075b68ed 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -180,7 +180,7 @@ class RandomSnTest(unittest.TestCase): def setUp(self): self.cert_count = 5 - self.serial_num = [] # type: List[int] + self.serial_num: List[int] = [] self.key = OpenSSL.crypto.PKey() self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index 74d1737ec..99a3a9ce4 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -1,4 +1,5 @@ """Tests for acme.messages.""" +from typing import Dict import unittest from unittest import mock @@ -81,7 +82,7 @@ class ConstantTest(unittest.TestCase): from acme.messages import _Constant class MockConstant(_Constant): # pylint: disable=missing-docstring - POSSIBLE_NAMES = {} # type: Dict + POSSIBLE_NAMES: Dict = {} self.MockConstant = MockConstant # pylint: disable=invalid-name self.const_a = MockConstant('a') diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index e6aa8f2d6..a02d008a0 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -41,7 +41,7 @@ class HTTP01ServerTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() # type: Set + self.resources: Set = set() from acme.standalone import HTTP01Server self.server = HTTP01Server(('', 0), resources=self.resources) @@ -218,7 +218,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() # type: Set + self.resources: Set = set() from acme.standalone import HTTP01DualNetworkedServers self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources) diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index 461b5cb7b..4549e935a 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -355,7 +355,7 @@ class AugeasBlockNode(AugeasDirectiveNode): ownpath = self.metadata.get("augeaspath") directives = self.parser.find_dir(name, start=ownpath, exclude=exclude) - already_parsed = set() # type: Set[str] + already_parsed: Set[str] = set() for directive in directives: # Remove the /arg part from the Augeas path directive = directive.partition("/arg")[0] diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index ecd5c46de..29648d4c1 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -16,11 +16,6 @@ from typing import Union import zope.component import zope.interface -try: - import apacheconfig - HAS_APACHECONFIG = True -except ImportError: # pragma: no cover - HAS_APACHECONFIG = False from acme import challenges from certbot import errors @@ -41,6 +36,12 @@ from certbot_apache._internal import http_01 from certbot_apache._internal import obj from certbot_apache._internal import parser +try: + import apacheconfig + HAS_APACHECONFIG = True +except ImportError: # pragma: no cover + HAS_APACHECONFIG = False + logger = logging.getLogger(__name__) @@ -211,23 +212,23 @@ class ApacheConfigurator(common.Installer): super(ApacheConfigurator, self).__init__(*args, **kwargs) # Add name_server association dict - self.assoc = {} # type: Dict[str, obj.VirtualHost] + self.assoc: Dict[str, obj.VirtualHost] = {} # Outstanding challenges - self._chall_out = set() # type: Set[KeyAuthorizationAnnotatedChallenge] + self._chall_out: Set[KeyAuthorizationAnnotatedChallenge] = set() # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {} # Maps enhancements to vhosts we've enabled the enhancement for - self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] + self._enhanced_vhosts: DefaultDict[str, Set[obj.VirtualHost]] = defaultdict(set) # Temporary state for AutoHSTS enhancement - self._autohsts = {} # type: Dict[str, Dict[str, Union[int, float]]] + self._autohsts: Dict[str, Dict[str, Union[int, float]]] = {} # Reverter save notes self.save_notes = "" # Should we use ParserNode implementation instead of the old behavior self.USE_PARSERNODE = use_parsernode # Saves the list of file paths that were parsed initially, and # not added to parser tree by self.conf("vhost-root") for example. - self.parsed_paths = [] # type: List[str] + self.parsed_paths: List[str] = [] # These will be set in the prepare function self._prepared = False self.parser = None @@ -833,7 +834,7 @@ class ApacheConfigurator(common.Installer): :rtype: set """ - all_names = set() # type: Set[str] + all_names: Set[str] = set() vhost_macro = [] @@ -997,8 +998,8 @@ class ApacheConfigurator(common.Installer): """ # Search base config, and all included paths for VirtualHosts - file_paths = {} # type: Dict[str, str] - internal_paths = defaultdict(set) # type: DefaultDict[str, Set[str]] + file_paths: Dict[str, str] = {} + internal_paths: DefaultDict[str, Set[str]] = defaultdict(set) vhs = [] # Make a list of parser paths because the parser_paths # dictionary may be modified during the loop. @@ -2157,7 +2158,7 @@ class ApacheConfigurator(common.Installer): # There can be other RewriteRule directive lines in vhost config. # rewrite_args_dict keys are directive ids and the corresponding value # for each is a list of arguments to that directive. - rewrite_args_dict = defaultdict(list) # type: DefaultDict[str, List[str]] + rewrite_args_dict: DefaultDict[str, List[str]] = defaultdict(list) pat = r'(.*directive\[\d+\]).*' for match in rewrite_path: m = re.match(pat, match) @@ -2251,7 +2252,7 @@ class ApacheConfigurator(common.Installer): if ssl_vhost.aliases: serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases) - rewrite_rule_args = [] # type: List[str] + rewrite_rule_args: List[str] = [] if self.get_version() >= (2, 3, 9): rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END else: diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 4bd935d2d..66d888e97 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -57,7 +57,7 @@ class ApacheHttp01(common.ChallengePerformer): self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") - self.moded_vhosts = set() # type: Set[VirtualHost] + self.moded_vhosts: Set[VirtualHost] = set() def perform(self): """Perform all HTTP-01 challenges.""" @@ -93,7 +93,7 @@ class ApacheHttp01(common.ChallengePerformer): self.configurator.enable_mod(mod, temp=True) def _mod_config(self): - selected_vhosts = [] # type: List[VirtualHost] + selected_vhosts: List[VirtualHost] = [] http_port = str(self.configurator.config.http01_port) for chall in self.achalls: # Search for matching VirtualHosts diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 5e4680e1b..21fd042f6 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -137,7 +137,7 @@ class VirtualHost: def get_names(self): """Return a set of all names.""" - all_names = set() # type: Set[str] + all_names: Set[str] = set() all_names.update(self.aliases) # Strip out any scheme:// and field from servername if self.name is not None: @@ -245,7 +245,7 @@ class VirtualHost: # already_found acts to keep everything very conservative. # Don't allow multiple ip:ports in same set. - already_found = set() # type: Set[str] + already_found: Set[str] = set() for addr in vhost.addrs: for local_addr in self.addrs: diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 4cde65d57..54edfc911 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -102,9 +102,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator): loadmods = self.parser.find_dir("LoadModule", "ssl_module", exclude=False) - correct_ifmods = [] # type: List[str] - loadmod_args = [] # type: List[str] - loadmod_paths = [] # type: List[str] + correct_ifmods: List[str] = [] + loadmod_args: List[str] = [] + loadmod_paths: List[str] = [] for m in loadmods: noarg_path = m.rpartition("/")[0] path_args = self.parser.get_all_args(noarg_path) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index eb8f267f9..1a6af4c4b 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -6,7 +6,6 @@ import re from typing import Dict from typing import List - from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util @@ -49,9 +48,9 @@ class ApacheParser: "version 1.2.0 or higher, please make sure you have you have " "those installed.") - self.modules = {} # type: Dict[str, str] - self.parser_paths = {} # type: Dict[str, List[str]] - self.variables = {} # type: Dict[str, str] + self.modules: Dict[str, str] = {} + self.parser_paths: Dict[str, List[str]] = {} + self.variables: Dict[str, str] = {} # Find configuration root and make sure augeas can parse it. self.root = os.path.abspath(root) @@ -264,7 +263,7 @@ class ApacheParser: the iteration issue. Else... parse and enable mods at same time. """ - mods = {} # type: Dict[str, str] + mods: Dict[str, str] = {} matches = self.find_dir("LoadModule") iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration @@ -551,7 +550,7 @@ class ApacheParser: else: arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg) - ordered_matches = [] # type: List[str] + ordered_matches: List[str] = [] # TODO: Wildcards should be included in alphabetical order # https://httpd.apache.org/docs/2.4/mod/core.html#include diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index abe72a5d0..83224fc98 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -107,7 +107,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- def test_set_parameters(self): servernames = self.config.parser_root.find_directives("servername") - names = [] # type: List[str] + names: List[str] = [] for servername in servernames: names += servername.parameters self.assertFalse("going_to_set_this" in names) diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 696cd4a54..06e19e445 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -26,7 +26,7 @@ class ApacheHttp01Test(util.ApacheTest): super(ApacheHttp01Test, self).setUp(*args, **kwargs) self.account_key = self.rsa512jwk - self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] + self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = [] vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") # Takes the vhosts for encryption-example.demo, certbot.demo diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py index 4c746e1ee..2c52c8523 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -32,8 +32,8 @@ def test_context(request): '--preferred-challenges', 'http' ], {'default_server': False}), ], indirect=['context']) -def test_certificate_deployment(certname_pattern, params, context): - # type: (str, List[str], nginx_context.IntegrationTestsContext) -> None +def test_certificate_deployment(certname_pattern: str, params: List[str], + context: nginx_context.IntegrationTestsContext) -> None: """ Test various scenarios to deploy a certificate to nginx using certbot. """ diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 706c3ebae..ceebbe7ed 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -51,7 +51,7 @@ class ACMEServer: self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' self._proxy = http_proxy self._workspace = tempfile.mkdtemp() - self._processes = [] # type: List[subprocess.Popen] + self._processes: List[subprocess.Popen] = [] self._stdout = sys.stdout if stdout else open(os.devnull, 'w') self._dns_server = dns_server self._http_01_port = http_01_port diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index eaa601f1b..31e7aee18 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -38,7 +38,7 @@ class DNSServer: self.bind_root = tempfile.mkdtemp() - self.process = None # type: subprocess.Popen + self.process: subprocess.Popen = None self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]} @@ -111,8 +111,7 @@ class DNSServer: self.stop() raise - def _wait_until_ready(self, attempts=30): - # type: (int) -> None + def _wait_until_ready(self, attempts: int = 30) -> None: """ Polls the DNS server over TCP until it gets a response, or until it runs out of attempts and raises a ValueError. diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py index 58b72c205..d7b7a120e 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py @@ -68,7 +68,7 @@ def _get_server_root(config): def _get_names(config): """Returns all and testable domain names in config""" - all_names = set() # type: Set[str] + all_names: Set[str] = set() for root, _dirs, files in os.walk(config): for this_file in files: update_names = _get_server_names(root, this_file) diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index fe634ba9f..4bb16e63f 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -178,7 +178,7 @@ def test_enhancements(plugin, domains): "enhancements") return False - domains_and_info = [(domain, []) for domain in domains] # type: List[Tuple[str, List[bool]]] + domains_and_info: List[Tuple[str, List[bool]]] = [(domain, []) for domain in domains] for domain, info in domains_and_info: try: diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index 583b41410..034d7bbb2 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -172,7 +172,7 @@ class _CloudflareClient: """ zone_name_guesses = dns_common.base_domain_name_guesses(domain) - zones = [] # type: List[Dict[str, Any]] + zones: List[Dict[str, Any]] = [] code = msg = None for zone_name in zone_name_guesses: diff --git a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py index 4837e2036..4dda13f1d 100644 --- a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py @@ -39,7 +39,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) self.r53 = boto3.client("route53") - self._resource_records = collections.defaultdict(list) # type: DefaultDict[str, List[Dict[str, str]]] + self._resource_records: DefaultDict[str, List[Dict[str, str]]] = collections.defaultdict(list) def more_info(self): # pylint: disable=missing-function-docstring return "Solve a DNS01 challenge using AWS Route53" diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 33baf076a..aab489315 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -112,8 +112,8 @@ class NginxConfigurator(common.Installer): # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] - self._wildcard_redirect_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {} + self._wildcard_redirect_vhosts: Dict[str, List[obj.VirtualHost]] = {} # Add number of outstanding challenges self._chall_out = 0 @@ -641,7 +641,7 @@ class NginxConfigurator(common.Installer): :rtype: set """ - all_names = set() # type: Set[str] + all_names: Set[str] = set() for vhost in self.parser.get_vhosts(): try: @@ -1222,7 +1222,7 @@ def nginx_restart(nginx_ctl, nginx_conf, sleep_duration): """ try: - reload_output = u"" # type: Text + reload_output: Text = u"" with tempfile.TemporaryFile() as out: proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"], env=util.env_no_snap_for_external_calls(), diff --git a/certbot-nginx/certbot_nginx/_internal/constants.py b/certbot-nginx/certbot_nginx/_internal/constants.py index 7b4111577..52f4d8238 100644 --- a/certbot-nginx/certbot_nginx/_internal/constants.py +++ b/certbot-nginx/certbot_nginx/_internal/constants.py @@ -14,11 +14,11 @@ elif platform.system() in ('NetBSD',): else: server_root_tmp = LINUX_SERVER_ROOT -CLI_DEFAULTS = dict( +CLI_DEFAULTS: Dict[str, Any] = dict( server_root=server_root_tmp, ctl="nginx", sleep_seconds=1 -) # type: Dict[str, Any] +) """CLI defaults.""" diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index ec610f15b..abcdfbd53 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -113,7 +113,7 @@ class NginxHttp01(common.ChallengePerformer): :returns: list of :class:`certbot_nginx._internal.obj.Addr` to apply :rtype: list """ - addresses = [] # type: List[obj.Addr] + addresses: List[obj.Addr] = [] default_addr = "%s" % self.configurator.config.http01_port ipv6_addr = "[::]:{0}".format( self.configurator.config.http01_port) diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index 1aee1f329..787f7c363 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -2,8 +2,8 @@ # Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed) import copy import logging -from typing import Any # pylint: disable=unused-import -from typing import IO # pylint: disable=unused-import +from typing import Any +from typing import IO from pyparsing import Combine from pyparsing import Forward @@ -20,6 +20,7 @@ from pyparsing import ZeroOrMore logger = logging.getLogger(__name__) + class RawNginxParser: # pylint: disable=pointless-statement """A class that parses nginx configuration with pyparsing.""" @@ -105,57 +106,9 @@ class RawNginxDumper: return ''.join(self) -# Shortcut functions to respect Python's serialization interface -# (like pyyaml, picker or json) - -def loads(source): - """Parses from a string. - - :param str source: The string to parse - :returns: The parsed tree - :rtype: list - - """ - return UnspacedList(RawNginxParser(source).as_list()) - - -def load(_file): - """Parses from a file. - - :param file _file: The file to parse - :returns: The parsed tree - :rtype: list - - """ - return loads(_file.read()) - - -def dumps(blocks): - # type: (UnspacedList) -> str - """Dump to a Unicode string. - - :param UnspacedList block: The parsed tree - :rtype: str - - """ - return str(RawNginxDumper(blocks.spaced)) - - -def dump(blocks, _file): - # type: (UnspacedList, IO[Any]) -> None - """Dump to a file. - - :param UnspacedList block: The parsed tree - :param IO[Any] _file: The file stream to dump to. It must be opened with - Unicode encoding. - :rtype: None - - """ - _file.write(dumps(blocks)) - - spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == '' + class UnspacedList(list): """Wrap a list [of lists], making any whitespace entries magically invisible""" @@ -274,3 +227,50 @@ class UnspacedList(list): idx -= 1 pos += 1 return idx0 + spaces + + +# Shortcut functions to respect Python's serialization interface +# (like pyyaml, picker or json) + +def loads(source): + """Parses from a string. + + :param str source: The string to parse + :returns: The parsed tree + :rtype: list + + """ + return UnspacedList(RawNginxParser(source).as_list()) + + +def load(_file): + """Parses from a file. + + :param file _file: The file to parse + :returns: The parsed tree + :rtype: list + + """ + return loads(_file.read()) + + +def dumps(blocks: UnspacedList) -> str: + """Dump to a Unicode string. + + :param UnspacedList block: The parsed tree + :rtype: six.text_type + + """ + return str(RawNginxDumper(blocks.spaced)) + + +def dump(blocks: UnspacedList, _file: IO[Any]) -> None: + """Dump to a file. + + :param UnspacedList block: The parsed tree + :param IO[Any] _file: The file stream to dump to. It must be opened with + Unicode encoding. + :rtype: None + + """ + _file.write(dumps(blocks)) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index ed7e211d5..8b353c095 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -32,7 +32,7 @@ class NginxParser: """ def __init__(self, root): - self.parsed = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] + self.parsed: Dict[str, Union[List, nginxparser.UnspacedList]] = {} self.root = os.path.abspath(root) self.config_root = self._find_config_root() @@ -94,7 +94,7 @@ class NginxParser: """ servers = self._get_raw_servers() - addr_to_ssl = {} # type: Dict[Tuple[str, str], bool] + addr_to_ssl: Dict[Tuple[str, str], bool] = {} for filename in servers: for server, _ in servers[filename]: # Parse the server block to save addr info @@ -106,12 +106,11 @@ class NginxParser: addr_to_ssl[addr_tuple] = addr.ssl or addr_to_ssl[addr_tuple] return addr_to_ssl - def _get_raw_servers(self): + def _get_raw_servers(self) -> Dict: # pylint: disable=cell-var-from-loop - # type: () -> Dict """Get a map of unparsed all server blocks """ - servers = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] + servers: Dict[str, Union[List, nginxparser.UnspacedList]] = {} for filename in self.parsed: tree = self.parsed[filename] servers[filename] = [] @@ -741,9 +740,9 @@ def _parse_server_raw(server): :rtype: dict """ - addrs = set() # type: Set[obj.Addr] - ssl = False # type: bool - names = set() # type: Set[str] + addrs: Set[obj.Addr] = set() + ssl: bool = False + names: Set[str] = set() apply_ssl_to_all_addrs = False diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 9efd6651a..37392ea80 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -5,7 +5,6 @@ import abc import logging from typing import List - from certbot import errors logger = logging.getLogger(__name__) @@ -23,7 +22,7 @@ class Parsable: __metaclass__ = abc.ABCMeta def __init__(self, parent=None): - self._data = [] # type: List[object] + self._data: List[object] = [] self._tabs = None self.parent = parent @@ -182,7 +181,7 @@ class Statements(Parsable): def _space_list(list_): """ Inserts whitespace between adjacent non-whitespace tokens. """ - spaced_statement = [] # type: List[str] + spaced_statement: List[str] = [] for i in reversed(range(len(list_))): spaced_statement.insert(0, list_[i]) if i > 0 and not list_[i].isspace() and not list_[i-1].isspace(): @@ -271,8 +270,8 @@ class Block(Parsable): """ def __init__(self, parent=None): super(Block, self).__init__(parent) - self.names = None # type: Sentence - self.contents = None # type: Block + self.names: Sentence = None + self.contents: Block = None @staticmethod def should_parse(lists): diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 23fe390ad..35173d094 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -112,7 +112,7 @@ class NginxParserTest(util.NginxTest): ([[[0], [3], [4]], [[5], [3], [0]]], [])] for mylist, result in mylists: - paths = [] # type: List[List[int]] + paths: List[List[int]] = [] parser._do_for_subarray(mylist, lambda x: isinstance(x, list) and len(x) >= 1 and diff --git a/certbot/certbot/_internal/account.py b/certbot/certbot/_internal/account.py index 07ae95e9d..c5667a865 100644 --- a/certbot/certbot/_internal/account.py +++ b/certbot/certbot/_internal/account.py @@ -229,8 +229,7 @@ class AccountFileStorage(interfaces.AccountStorage): def load(self, account_id): return self._load_for_server_path(account_id, self.config.server_path) - def save(self, account, client): - # type: (Account, ClientBase) -> None + def save(self, account: Account, client: ClientBase) -> None: """Create a new account. :param Account account: account to create @@ -245,8 +244,7 @@ class AccountFileStorage(interfaces.AccountStorage): except IOError as error: raise errors.AccountStorageError(error) - def update_regr(self, account, client): - # type: (Account, ClientBase) -> None + def update_regr(self, account: Account, client: ClientBase) -> None: """Update the registration resource. :param Account account: account to update @@ -259,8 +257,7 @@ class AccountFileStorage(interfaces.AccountStorage): except IOError as error: raise errors.AccountStorageError(error) - def update_meta(self, account): - # type: (Account) -> None + def update_meta(self, account: Account) -> None: """Update the meta resource. :param Account account: account to update @@ -338,19 +335,16 @@ class AccountFileStorage(interfaces.AccountStorage): return dir_path - def _prepare(self, account): - # type: (Account) -> str + def _prepare(self, account: Account) -> str: account_dir_path = self._account_dir_path(account.id) util.make_or_verify_dir(account_dir_path, 0o700, self.config.strict_permissions) return account_dir_path - def _create(self, account, dir_path): - # type: (Account, str) -> None + def _create(self, account: Account, dir_path: str) -> None: with util.safe_open(self._key_path(dir_path), "w", chmod=0o400) as key_file: key_file.write(account.key.json_dumps()) - def _update_regr(self, account, acme, dir_path): - # type: (Account, ClientBase, str) -> None + def _update_regr(self, account: Account, acme: ClientBase, dir_path: str) -> None: with open(self._regr_path(dir_path), "w") as regr_file: regr = account.regr # If we have a value for new-authz, save it for forwards diff --git a/certbot/certbot/_internal/auth_handler.py b/certbot/certbot/_internal/auth_handler.py index 2801e3c2f..c2f323a36 100644 --- a/certbot/certbot/_internal/auth_handler.py +++ b/certbot/certbot/_internal/auth_handler.py @@ -98,8 +98,7 @@ class AuthHandler: return authzrs_validated - def deactivate_valid_authorizations(self, orderr): - # type: (messages.OrderResource) -> Tuple[List, List] + def deactivate_valid_authorizations(self, orderr: messages.OrderResource) -> Tuple[List, List]: """ Deactivate all `valid` authorizations in the order, so that they cannot be re-used in subsequent orders. @@ -191,7 +190,7 @@ class AuthHandler: """ pending_authzrs = [authzr for authzr in authzrs if authzr.body.status != messages.STATUS_VALID] - achalls = [] # type: List[achallenges.AnnotatedChallenge] + achalls: List[achallenges.AnnotatedChallenge] = [] if pending_authzrs: logger.info("Performing the following challenges:") for authzr in pending_authzrs: @@ -428,7 +427,7 @@ _ERROR_HELP = { def _report_failed_authzrs(failed_authzrs, account_key): """Notifies the user about failed authorizations.""" - problems = {} # type: Dict[str, List[achallenges.AnnotatedChallenge]] + problems: Dict[str, List[achallenges.AnnotatedChallenge]] = {} failed_achalls = [challb_to_achall(challb, account_key, authzr.body.identifier.value) for authzr in failed_authzrs for challb in authzr.body.challenges if challb.error] diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index 2ac9fc538..8fab5735a 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -241,7 +241,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func def find_matches(candidate_lineage, return_value, acceptable_matches): """Returns a list of matches using _search_lineages.""" acceptable_matches = [func(candidate_lineage) for func in acceptable_matches] - acceptable_matches_rv = [] # type: List[str] + acceptable_matches_rv: List[str] = [] for item in acceptable_matches: if isinstance(item, list): acceptable_matches_rv += item @@ -364,7 +364,7 @@ def _report_human_readable(config, parsed_certs): def _describe_certs(config, parsed_certs, parse_failures): """Print information about the certs we know about""" - out = [] # type: List[str] + out: List[str] = [] notify = out.append diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index 30f444e5b..b64ab0e9c 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -66,7 +66,7 @@ logger = logging.getLogger(__name__) # Global, to save us from a lot of argument passing within the scope of this module -helpful_parser = None # type: Optional[HelpfulArgumentParser] +helpful_parser: Optional[HelpfulArgumentParser] = None def prepare_and_parse_args(plugins, args, detect_defaults=False): diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index f1f9d0f3c..1f30261c1 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -62,7 +62,7 @@ def config_help(name, hidden=False): """Extract the help message for an `.IConfig` attribute.""" if hidden: return argparse.SUPPRESS - field = interfaces.IConfig.__getitem__(name) # type: zope.interface.interface.Attribute + field: zope.interface.interface.Attribute = interfaces.IConfig.__getitem__(name) return field.__doc__ diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index cbac4e232..d36147089 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -105,9 +105,9 @@ class HelpfulArgumentParser: self.visible_topics = self.determine_help_topics(self.help_arg) # elements are added by .add_group() - self.groups = {} # type: Dict[str, argparse._ArgumentGroup] + self.groups: Dict[str, argparse._ArgumentGroup] = {} # elements are added by .parse_args() - self.defaults = {} # type: Dict[str, Any] + self.defaults: Dict[str, Any] = {} self.parser = configargparse.ArgParser( prog="certbot", diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index b9137e249..ba696bf85 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -328,7 +328,7 @@ class Client: with open(old_keypath, "rb") as f: keypath = old_keypath keypem = f.read() - key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key] + key: Optional[util.Key] = util.Key(file=keypath, pem=keypem) logger.info("Reusing existing private key from %s.", old_keypath) else: # The key is set to None here but will be created below. @@ -390,8 +390,8 @@ class Client: cert, chain = self.obtain_certificate_from_csr(csr, orderr) return cert, chain, key, csr - def _get_order_and_authorizations(self, csr_pem, best_effort): - # type: (str, bool) -> List[messages.OrderResource] + def _get_order_and_authorizations(self, csr_pem: str, + best_effort: bool) -> List[messages.OrderResource]: """Request a new order and complete its authorizations. :param str csr_pem: A CSR in PEM format. diff --git a/certbot/certbot/_internal/eff.py b/certbot/certbot/_internal/eff.py index 6212c24ee..b01e2dd61 100644 --- a/certbot/certbot/_internal/eff.py +++ b/certbot/certbot/_internal/eff.py @@ -1,22 +1,21 @@ """Subscribes users to the EFF newsletter.""" import logging -from typing import Optional # pylint: disable=unused-import +from typing import Optional import requests import zope.component from certbot import interfaces from certbot._internal import constants -from certbot._internal.account import Account # pylint: disable=unused-import +from certbot._internal.account import Account from certbot._internal.account import AccountFileStorage from certbot.display import util as display_util -from certbot.interfaces import IConfig # pylint: disable=unused-import +from certbot.interfaces import IConfig logger = logging.getLogger(__name__) -def prepare_subscription(config, acc): - # type: (IConfig, Account) -> None +def prepare_subscription(config: IConfig, acc: Account) -> None: """High level function to store potential EFF newsletter subscriptions. The user may be asked if they want to sign up for the newsletter if @@ -44,8 +43,7 @@ def prepare_subscription(config, acc): storage.update_meta(acc) -def handle_subscription(config, acc): - # type: (IConfig, Account) -> None +def handle_subscription(config: IConfig, acc: Account) -> None: """High level function to take care of EFF newsletter subscriptions. Once subscription is handled, it will not be handled again. @@ -64,8 +62,7 @@ def handle_subscription(config, acc): storage.update_meta(acc) -def _want_subscription(): - # type: () -> bool +def _want_subscription() -> bool: """Does the user want to be subscribed to the EFF newsletter? :returns: True if we should subscribe the user, otherwise, False @@ -82,8 +79,7 @@ def _want_subscription(): return display.yesno(prompt, default=False) -def subscribe(email): - # type: (str) -> None +def subscribe(email: str) -> None: """Subscribe the user to the EFF mailing list. :param str email: the e-mail address to subscribe @@ -98,8 +94,7 @@ def subscribe(email): _check_response(requests.post(url, data=data)) -def _check_response(response): - # type: (requests.Response) -> None +def _check_response(response: requests.Response) -> None: """Check for errors in the server's response. If an error occurred, it will be reported to the user. @@ -119,8 +114,7 @@ def _check_response(response): _report_failure('there was a problem with the server response') -def _report_failure(reason=None): - # type: (Optional[str]) -> None +def _report_failure(reason: Optional[str] = None) -> None: """Notify the user of failing to sign them up for the newsletter. :param reason: a phrase describing what the problem was diff --git a/certbot/certbot/_internal/error_handler.py b/certbot/certbot/_internal/error_handler.py index cb8301980..01cc92b42 100644 --- a/certbot/certbot/_internal/error_handler.py +++ b/certbot/certbot/_internal/error_handler.py @@ -77,9 +77,9 @@ class ErrorHandler: def __init__(self, func, *args, **kwargs): self.call_on_regular_exit = False self.body_executed = False - self.funcs = [] # type: List[Callable[[], Any]] - self.prev_handlers = {} # type: Dict[int, Union[int, None, Callable]] - self.received_signals = [] # type: List[int] + self.funcs: List[Callable[[], Any]] = [] + self.prev_handlers: Dict[int, Union[int, None, Callable]] = {} + self.received_signals: List[int] = [] if func is not None: self.register(func, *args, **kwargs) @@ -108,8 +108,7 @@ class ErrorHandler: self._call_signals() return retval - def register(self, func, *args, **kwargs): - # type: (Callable, *Any, **Any) -> None + def register(self, func: Callable, *args: Any, **kwargs: Any) -> None: """Sets func to be run with the given arguments during cleanup. :param function func: function to be called in case of an error diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index b695d9930..b9f1f1531 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -77,7 +77,7 @@ def pre_hook(config): _run_pre_hook_if_necessary(cmd) -executed_pre_hooks = set() # type: Set[str] +executed_pre_hooks: Set[str] = set() def _run_pre_hook_if_necessary(command): @@ -127,7 +127,7 @@ def post_hook(config): _run_hook("post-hook", cmd) -post_hooks = [] # type: List[str] +post_hooks: List[str] = [] def _run_eventually(command): diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index f878946cb..d00302598 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -16,28 +16,9 @@ else: POSIX_MODE = True - logger = logging.getLogger(__name__) -def lock_dir(dir_path): - # type: (str) -> LockFile - """Place a lock file on the directory at dir_path. - - The lock file is placed in the root of dir_path with the name - .certbot.lock. - - :param str dir_path: path to directory - - :returns: the locked LockFile object - :rtype: LockFile - - :raises errors.LockError: if unable to acquire the lock - - """ - return LockFile(os.path.join(dir_path, '.certbot.lock')) - - class LockFile: """ Platform independent file lock system. @@ -52,8 +33,7 @@ class LockFile: LockFile is platform independent: it will proceed to the appropriate OS lock mechanism depending on Linux or Windows. """ - def __init__(self, path): - # type: (str) -> None + def __init__(self, path: str) -> None: """ Create a LockFile instance on the given file path, and acquire lock. :param str path: the path to the file that will hold a lock @@ -64,8 +44,7 @@ class LockFile: self.acquire() - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) if self.is_locked(): repr_str += 'acquired>' @@ -73,23 +52,20 @@ class LockFile: repr_str += 'released>' return repr_str - def acquire(self): - # type: () -> None + def acquire(self) -> None: """ Acquire the lock on the file, forbidding any other Certbot instance to acquire it. :raises errors.LockError: if unable to acquire the lock """ self._lock_mechanism.acquire() - def release(self): - # type: () -> None + def release(self) -> None: """ Release the lock on the file, allowing any other Certbot instance to acquire it. """ self._lock_mechanism.release() - def is_locked(self): - # type: () -> bool + def is_locked(self) -> bool: """ Check if the file is currently locked. :return: True if the file is locked, False otherwise @@ -98,17 +74,15 @@ class LockFile: class _BaseLockMechanism: - def __init__(self, path): - # type: (str) -> None + def __init__(self, path: str) -> None: """ Create a lock file mechanism for Unix. :param str path: the path to the lock file """ self._path = path - self._fd = None # type: Optional[int] + self._fd: Optional[int] = None - def is_locked(self): - # type: () -> bool + def is_locked(self) -> bool: """Check if lock file is currently locked. :return: True if the lock file is locked :rtype: bool @@ -129,8 +103,7 @@ class _UnixLockMechanism(_BaseLockMechanism): process exits. It cannot be used to provide synchronization between threads. It is based on the lock_file package by Martin Horcicka. """ - def acquire(self): - # type: () -> None + def acquire(self) -> None: """Acquire the lock.""" while self._fd is None: # Open the file @@ -144,8 +117,7 @@ class _UnixLockMechanism(_BaseLockMechanism): if self._fd is None: os.close(fd) - def _try_lock(self, fd): - # type: (int) -> None + def _try_lock(self, fd: int) -> None: """ Try to acquire the lock file without blocking. :param int fd: file descriptor of the opened file to lock @@ -158,8 +130,7 @@ class _UnixLockMechanism(_BaseLockMechanism): raise errors.LockError('Another instance of Certbot is already running.') raise - def _lock_success(self, fd): - # type: (int) -> bool + def _lock_success(self, fd: int) -> bool: """ Did we successfully grab the lock? Because this class deletes the locked file when the lock is @@ -185,8 +156,7 @@ class _UnixLockMechanism(_BaseLockMechanism): # the same device and inode, they're the same file. return stat1.st_dev == stat2.st_dev and stat1.st_ino == stat2.st_ino - def release(self): - # type: () -> None + def release(self) -> None: """Remove, close, and release the lock file.""" # It is important the lock file is removed before it's released, # otherwise: @@ -269,3 +239,20 @@ class _WindowsLockMechanism(_BaseLockMechanism): logger.debug(str(e)) finally: self._fd = None + + +def lock_dir(dir_path: str) -> LockFile: + """Place a lock file on the directory at dir_path. + + The lock file is placed in the root of dir_path with the name + .certbot.lock. + + :param str dir_path: path to directory + + :returns: the locked LockFile object + :rtype: LockFile + + :raises errors.LockError: if unable to acquire the lock + + """ + return LockFile(os.path.join(dir_path, '.certbot.lock')) diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index ba8d94bca..ac4d18449 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -4,7 +4,7 @@ import functools import logging.handlers import sys -from typing import Iterable # pylint: disable=unused-import +from typing import Iterable from typing import List from typing import Optional from typing import Tuple @@ -145,8 +145,8 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N return lineage -def _handle_unexpected_key_type_migration(config, cert): - # type: (configuration.NamespaceConfig, storage.RenewableCert) -> None +def _handle_unexpected_key_type_migration(config: configuration.NamespaceConfig, + cert: storage.RenewableCert) -> None: """ This function ensures that the user will not implicitly migrate an existing key from one type to another in the situation where a certificate for that lineage @@ -167,11 +167,10 @@ def _handle_unexpected_key_type_migration(config, cert): raise errors.Error(msg) -def _handle_subset_cert_request(config, # type: configuration.NamespaceConfig - domains, # type: List[str] - cert # type: storage.RenewableCert - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _handle_subset_cert_request(config: configuration.NamespaceConfig, + domains: List[str], + cert: storage.RenewableCert + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Figure out what to do if a previous cert had a subset of the names now requested :param config: Configuration object @@ -218,10 +217,9 @@ def _handle_subset_cert_request(config, # type: configuration.NamespaceConfig raise errors.Error(USER_CANCELLED) -def _handle_identical_cert_request(config, # type: configuration.NamespaceConfig - lineage, # type: storage.RenewableCert - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _handle_identical_cert_request(config: configuration.NamespaceConfig, + lineage: storage.RenewableCert, + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Figure out what to do if a lineage has the same names as a previously obtained one :param config: Configuration object @@ -337,11 +335,10 @@ def _find_cert(config, domains, certname): return (action != "reinstall"), lineage -def _find_lineage_for_domains_and_certname(config, # type: configuration.NamespaceConfig - domains, # type: List[str] - certname # type: str - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _find_lineage_for_domains_and_certname(config: configuration.NamespaceConfig, + domains: List[str], + certname: str + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Find appropriate lineage based on given domains and/or certname. :param config: Configuration object @@ -758,7 +755,7 @@ def update_account(config, unused_plugins): cb_client = client.Client(config, acc, None, None, acme=acme) # Empty list of contacts in case the user is removing all emails - acc_contacts = () # type: Iterable[str] + acc_contacts: Iterable[str] = () if config.email: acc_contacts = ['mailto:' + email for email in config.email.split(',')] # We rely on an exception to interrupt this process if it didn't work. @@ -1362,8 +1359,8 @@ def set_displayer(config): """ if config.quiet: config.noninteractive_mode = True - displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) \ - # type: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] + displayer: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] =\ + display_util.NoninteractiveDisplay(open(os.devnull, "w")) elif config.noninteractive_mode: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index 27b2605e0..4097544e8 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -1,9 +1,9 @@ """Utilities for plugins discovery and selection.""" +from collections.abc import Mapping import itertools import logging import sys from typing import Dict -from collections.abc import Mapping import pkg_resources import zope.interface @@ -213,7 +213,7 @@ class PluginsRegistry(Mapping): @classmethod def find_all(cls): """Find plugins using setuptools entry points.""" - plugins = {} # type: Dict[str, PluginEntryPoint] + plugins: Dict[str, PluginEntryPoint] = {} plugin_paths_string = os.getenv('CERTBOT_PLUGIN_PATH') plugin_paths = plugin_paths_string.split(':') if plugin_paths_string else [] # XXX should ensure this only happens once diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index c28e333b9..ed2e0559e 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -5,7 +5,7 @@ import zope.component import zope.interface from acme import challenges -from certbot import achallenges # pylint: disable=unused-import +from certbot import achallenges from certbot import errors from certbot import interfaces from certbot import reverter @@ -74,8 +74,7 @@ permitted by DNS standards.) super(Authenticator, self).__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() - self.env = {} \ - # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] + self.env: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] = {} self.subsequent_dns_challenge = False self.subsequent_any_challenge = False diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index f511c60e9..833651346 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -10,7 +10,7 @@ from typing import Set from typing import Tuple from typing import TYPE_CHECKING -import OpenSSL # pylint: disable=unused-import +import OpenSSL import zope.interface from acme import challenges @@ -42,7 +42,7 @@ class ServerManager: """ def __init__(self, certs, http_01_resources): - self._instances = {} # type: Dict[int, acme_standalone.BaseDualNetworkedServers] + self._instances: Dict[int, acme_standalone.BaseDualNetworkedServers] = {} self.certs = certs self.http_01_resources = http_01_resources @@ -122,15 +122,14 @@ class Authenticator(common.Plugin): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.served = collections.defaultdict(set) # type: ServedType + self.served: ServedType = collections.defaultdict(set) # Stuff below is shared across threads (i.e. servers read # values, main thread writes). Due to the nature of CPython's # GIL, the operations are safe, c.f. # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe - self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] - self.http_01_resources = set() \ - # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] + self.certs: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] = {} + self.http_01_resources: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] = set() self.servers = ServerManager(self.certs, self.http_01_resources) diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index 730241066..c5b436b65 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -12,10 +12,10 @@ import zope.component import zope.interface from acme import challenges -from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces from certbot._internal import cli +from certbot.achallenges import KeyAuthorizationAnnotatedChallenge as AnnotatedChallenge from certbot.compat import filesystem from certbot.compat import os from certbot.display import ops @@ -67,11 +67,10 @@ to serve all files under specified web root ({0}).""" def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.full_roots = {} # type: Dict[str, str] - self.performed = collections.defaultdict(set) \ - # type: DefaultDict[str, Set[achallenges.KeyAuthorizationAnnotatedChallenge]] + self.full_roots: Dict[str, str] = {} + self.performed: DefaultDict[str, Set[AnnotatedChallenge]] = collections.defaultdict(set) # stack of dirs successfully created by this authenticator - self._created_dirs = [] # type: List[str] + self._created_dirs: List[str] = [] def prepare(self): # pylint: disable=missing-function-docstring pass @@ -224,7 +223,7 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - not_removed = [] # type: List[str] + not_removed: List[str] = [] while self._created_dirs: path = self._created_dirs.pop() try: diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index a09edfa1e..f29709ef4 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -8,7 +8,7 @@ import sys import time import traceback from typing import List -from typing import Optional # pylint: disable=unused-import +from typing import Optional from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import ec @@ -22,7 +22,7 @@ from certbot import errors from certbot import interfaces from certbot import util from certbot._internal import cli -from certbot._internal import client # pylint: disable=unused-import +from certbot._internal import client from certbot._internal import constants from certbot._internal import hooks from certbot._internal import storage @@ -143,7 +143,7 @@ def _restore_plugin_configs(config, renewalparams): # longer defined, stored copies of that parameter will be # deserialized as strings by this logic even if they were # originally meant to be some other type. - plugin_prefixes = [] # type: List[str] + plugin_prefixes: List[str] = [] if renewalparams["authenticator"] == "webroot": _restore_webroot_config(config, renewalparams) else: @@ -290,7 +290,7 @@ def _restore_str(name, value): def should_renew(config, lineage): - "Return true if any of the circumstances for automatic renewal apply." + """Return true if any of the circumstances for automatic renewal apply.""" if config.renew_by_default: logger.debug("Auto-renewal forced with --force-renewal...") return True @@ -305,7 +305,7 @@ def should_renew(config, lineage): def _avoid_invalidating_lineage(config, lineage, original_server): - "Do not renew a valid cert with one from a staging server!" + """Do not renew a valid cert with one from a staging server!""" # Some lineages may have begun with --staging, but then had production # certificates added to them with open(lineage.cert) as the_file: @@ -323,8 +323,8 @@ def _avoid_invalidating_lineage(config, lineage, original_server): "unless you use the --break-my-certs flag!".format(names)) -def renew_cert(config, domains, le_client, lineage): - # type: (interfaces.IConfig, Optional[List[str]], client.Client, storage.RenewableCert) -> None +def renew_cert(config: interfaces.IConfig, domains: Optional[List[str]], le_client: client.Client, + lineage: storage.RenewableCert) -> None: """Renew a certificate lineage.""" renewal_params = lineage.configuration["renewalparams"] original_server = renewal_params.get("server", cli.flag_default("server")) @@ -351,14 +351,14 @@ def renew_cert(config, domains, le_client, lineage): def report(msgs, category): - "Format a results report for a category of renewal outcomes" + """Format a results report for a category of renewal outcomes""" lines = ("%s (%s)" % (m, category) for m in msgs) return " " + "\n ".join(lines) -def _renew_describe_results(config, renew_successes, renew_failures, - renew_skipped, parse_failures): - # type: (interfaces.IConfig, List[str], List[str], List[str], List[str]) -> None +def _renew_describe_results(config: interfaces.IConfig, renew_successes: List[str], + renew_failures: List[str], renew_skipped: List[str], + parse_failures: List[str]) -> None: """ Print a report to the terminal about the results of the renewal process. @@ -511,8 +511,7 @@ def handle_renewal_request(config): logger.debug("no renewal failures") -def _update_renewal_params_from_key(key_path, config): - # type: (str, interfaces.IConfig) -> None +def _update_renewal_params_from_key(key_path: str, config: interfaces.IConfig) -> None: with open(key_path, 'rb') as file_h: key = load_pem_private_key(file_h.read(), password=None, backend=default_backend()) if isinstance(key, rsa.RSAPrivateKey): diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index 3093ef8ca..0268fbf34 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -32,7 +32,7 @@ class Reporter: _msg_type = collections.namedtuple('ReporterMsg', 'priority text on_crash') def __init__(self, config): - self.messages = queue.PriorityQueue() # type: queue.PriorityQueue[Reporter._msg_type] + self.messages: queue.PriorityQueue[Reporter._msg_type] = queue.PriorityQueue() self.config = config def add_message(self, msg, priority, on_crash=True): diff --git a/certbot/certbot/_internal/snap_config.py b/certbot/certbot/_internal/snap_config.py index f2fe035db..1ba53cad4 100644 --- a/certbot/certbot/_internal/snap_config.py +++ b/certbot/certbot/_internal/snap_config.py @@ -33,8 +33,7 @@ _ARCH_TRIPLET_MAP = { LOGGER = logging.getLogger(__name__) -def prepare_env(cli_args): - # type: (List[str]) -> List[str] +def prepare_env(cli_args: List[str]) -> List[str]: """ Prepare runtime environment for a certbot execution in snap. :param list cli_args: List of command line arguments diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index aa6fe9134..25e1687da 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -35,8 +35,7 @@ class _WindowsUmask: _WINDOWS_UMASK = _WindowsUmask() -def chmod(file_path, mode): - # type: (str, int) -> None +def chmod(file_path: str, mode: int) -> None: """ Apply a POSIX mode on given file_path: @@ -57,8 +56,7 @@ def chmod(file_path, mode): _apply_win_mode(file_path, mode) -def umask(mask): - # type: (int) -> int +def umask(mask: int) -> int: """ Set the current numeric umask and return the previous umask. On Linux, the built-in umask method is used. On Windows, our Certbot-side implementation is used. @@ -84,8 +82,8 @@ def umask(mask): # Since copying and editing arbitrary DACL is very difficult, and since we actually know # the mode to apply at the time the owner of a file should change, it is easier to just # change the owner, then reapply the known mode, as copy_ownership_and_apply_mode() does. -def copy_ownership_and_apply_mode(src, dst, mode, copy_user, copy_group): - # type: (str, str, int, bool, bool) -> None +def copy_ownership_and_apply_mode(src: str, dst: str, mode: int, + copy_user: bool, copy_group: bool) -> None: """ Copy ownership (user and optionally group on Linux) from the source to the destination, then apply given mode in compatible way for Linux and Windows. @@ -117,8 +115,8 @@ def copy_ownership_and_apply_mode(src, dst, mode, copy_user, copy_group): # equivalent POSIX mode, because ownership and mode are copied altogether on the destination # file, so no recomputing of the DACL against the new owner is needed, as it would be # for a copy_ownership alone method. -def copy_ownership_and_mode(src, dst, copy_user=True, copy_group=True): - # type: (str, str, bool, bool) -> None +def copy_ownership_and_mode(src: str, dst: str, + copy_user: bool = True, copy_group: bool = True) -> None: """ Copy ownership (user and optionally group on Linux) and mode/DACL from the source to the destination. @@ -142,8 +140,7 @@ def copy_ownership_and_mode(src, dst, copy_user=True, copy_group=True): _copy_win_mode(src, dst) -def check_mode(file_path, mode): - # type: (str, int) -> bool +def check_mode(file_path: str, mode: int) -> bool: """ Check if the given mode matches the permissions of the given file. On Linux, will make a direct comparison, on Windows, mode will be compared against @@ -160,8 +157,7 @@ def check_mode(file_path, mode): return _check_win_mode(file_path, mode) -def check_owner(file_path): - # type: (str) -> bool +def check_owner(file_path: str) -> bool: """ Check if given file is owned by current user. @@ -183,8 +179,7 @@ def check_owner(file_path): return _get_current_user() == user -def check_permissions(file_path, mode): - # type: (str, int) -> bool +def check_permissions(file_path: str, mode: int) -> bool: """ Check if given file has the given mode and is owned by current user. @@ -196,8 +191,7 @@ def check_permissions(file_path, mode): return check_owner(file_path) and check_mode(file_path, mode) -def open(file_path, flags, mode=0o777): # pylint: disable=redefined-builtin - # type: (str, int, int) -> int +def open(file_path: str, flags: int, mode: int = 0o777) -> int: # pylint: disable=redefined-builtin """ Wrapper of original os.open function, that will ensure on Windows that given mode is correctly applied. @@ -266,8 +260,7 @@ def open(file_path, flags, mode=0o777): # pylint: disable=redefined-builtin return handle -def makedirs(file_path, mode=0o777): - # type: (str, int) -> None +def makedirs(file_path: str, mode: int = 0o777) -> None: """ Rewrite of original os.makedirs function, that will ensure on Windows that given mode is correctly applied. @@ -299,8 +292,7 @@ def makedirs(file_path, mode=0o777): umask(current_umask) -def mkdir(file_path, mode=0o777): - # type: (str, int) -> None +def mkdir(file_path: str, mode: int = 0o777) -> None: """ Rewrite of original os.mkdir function, that will ensure on Windows that given mode is correctly applied. @@ -331,8 +323,7 @@ def mkdir(file_path, mode=0o777): return None -def replace(src, dst): - # type: (str, str) -> None +def replace(src: str, dst: str) -> None: """ Rename a file to a destination path and handles situations where the destination exists. @@ -349,8 +340,7 @@ def replace(src, dst): os.rename(src, dst) -def realpath(file_path): - # type: (str) -> str +def realpath(file_path: str) -> str: """ Find the real path for the given path. This method resolves symlinks, including recursive symlinks, and is protected against symlinks that creates an infinite loop. @@ -371,7 +361,7 @@ def realpath(file_path): raise RuntimeError('Error, link {0} is a loop!'.format(original_path)) return path - inspected_paths = [] # type: List[str] + inspected_paths: List[str] = [] while os.path.islink(file_path): link_path = file_path file_path = os.readlink(file_path) @@ -384,8 +374,7 @@ def realpath(file_path): return os.path.abspath(file_path) -def readlink(link_path): - # type: (str) -> str +def readlink(link_path: str) -> str: """ Return a string representing the path to which the symbolic link points. @@ -419,8 +408,7 @@ def readlink(link_path): # elevated privileges or not. However this is not a problem since certbot always # requires to be run under a privileged shell, so the user will always benefit # from the highest (privileged one) set of permissions on a given file. -def is_executable(path): - # type: (str) -> bool +def is_executable(path: str) -> bool: """ Is path an executable file? @@ -434,8 +422,7 @@ def is_executable(path): return _win_is_executable(path) -def has_world_permissions(path): - # type: (str) -> bool +def has_world_permissions(path: str) -> bool: """ Check if everybody/world has any right (read/write/execute) on a file given its path. @@ -456,8 +443,7 @@ def has_world_permissions(path): })) -def compute_private_key_mode(old_key, base_mode): - # type: (str, int) -> int +def compute_private_key_mode(old_key: str, base_mode: int) -> int: """ Calculate the POSIX mode to apply to a private key given the previous private key. @@ -478,8 +464,7 @@ def compute_private_key_mode(old_key, base_mode): return base_mode -def has_same_ownership(path1, path2): - # type: (str, str) -> bool +def has_same_ownership(path1: str, path2: str) -> bool: """ Return True if the ownership of two files given their respective path is the same. On Windows, ownership is checked against owner only, since files do not have a group owner. @@ -504,8 +489,7 @@ def has_same_ownership(path1, path2): return user1 == user2 -def has_min_permissions(path, min_mode): - # type: (str, int) -> bool +def has_min_permissions(path: str, min_mode: int) -> bool: """ Check if a file given its path has at least the permissions defined by the given minimal mode. On Windows, group permissions are ignored since files do not have a group owner. diff --git a/certbot/certbot/compat/misc.py b/certbot/certbot/compat/misc.py index 297df80fc..3d2624906 100644 --- a/certbot/certbot/compat/misc.py +++ b/certbot/certbot/compat/misc.py @@ -27,8 +27,7 @@ logger = logging.getLogger(__name__) STANDARD_BINARY_DIRS = ["/usr/sbin", "/usr/local/bin", "/usr/local/sbin"] if POSIX_MODE else [] -def raise_for_non_administrative_windows_rights(): - # type: () -> None +def raise_for_non_administrative_windows_rights() -> None: """ On Windows, raise if current shell does not have the administrative rights. Do nothing on Linux. @@ -39,8 +38,7 @@ def raise_for_non_administrative_windows_rights(): raise errors.Error('Error, certbot must be run on a shell with administrative rights.') -def readline_with_timeout(timeout, prompt): - # type: (float, str) -> str +def readline_with_timeout(timeout: float, prompt: str) -> str: """ Read user input to return the first line entered, or raise after specified timeout. @@ -81,8 +79,7 @@ LINUX_DEFAULT_FOLDERS = { } -def get_default_folder(folder_type): - # type: (str) -> str +def get_default_folder(folder_type: str) -> str: """ Return the relevant default folder for the current OS @@ -99,8 +96,7 @@ def get_default_folder(folder_type): return WINDOWS_DEFAULT_FOLDERS[folder_type] -def underscores_for_unsupported_characters_in_path(path): - # type: (str) -> str +def underscores_for_unsupported_characters_in_path(path: str) -> str: """ Replace unsupported characters in path for current OS by underscores. :param str path: the path to normalize @@ -116,8 +112,7 @@ def underscores_for_unsupported_characters_in_path(path): return drive + tail.replace(':', '_') -def execute_command(cmd_name, shell_cmd, env=None): - # type: (str, str, Optional[dict]) -> Tuple[str, str] +def execute_command(cmd_name: str, shell_cmd: str, env: Optional[dict] = None) -> Tuple[str, str]: """ Run a command: - on Linux command will be run by the standard shell selected with Popen(shell=True) diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 5bc9251a4..445618ea0 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -7,7 +7,6 @@ import hashlib import logging import re -from typing import IO # pylint: disable=unused-import import warnings # See https://github.com/pyca/cryptography/issues/4275 @@ -272,9 +271,9 @@ def verify_renewable_cert_sig(renewable_cert): :raises errors.Error: If signature verification fails. """ try: - with open(renewable_cert.chain_path, 'rb') as chain_file: # type: IO[bytes] + with open(renewable_cert.chain_path, 'rb') as chain_file: chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) - with open(renewable_cert.cert_path, 'rb') as cert_file: # type: IO[bytes] + with open(renewable_cert.cert_path, 'rb') as cert_file: cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) pk = chain.public_key() with warnings.catch_warnings(): @@ -349,11 +348,11 @@ def verify_fullchain(renewable_cert): :raises errors.Error: If cert and chain do not combine to fullchain. """ try: - with open(renewable_cert.chain_path) as chain_file: # type: IO[str] + with open(renewable_cert.chain_path) as chain_file: chain = chain_file.read() - with open(renewable_cert.cert_path) as cert_file: # type: IO[str] + with open(renewable_cert.cert_path) as cert_file: cert = cert_file.read() - with open(renewable_cert.fullchain_path) as fullchain_file: # type: IO[str] + with open(renewable_cert.fullchain_path) as fullchain_file: fullchain = fullchain_file.read() if (cert + chain) != fullchain: error_str = "fullchain does not match cert + chain for {0}!" @@ -487,7 +486,7 @@ def _notAfterBefore(cert_path, method): """ # pylint: disable=redefined-outer-name - with open(cert_path, "rb") as f: # type: IO[bytes] + with open(cert_path, "rb") as f: x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) # pyopenssl always returns bytes timestamp = method(x509) @@ -564,7 +563,7 @@ def get_serial_from_cert(cert_path): :rtype: int """ # pylint: disable=redefined-outer-name - with open(cert_path, "rb") as f: # type: IO[bytes] + with open(cert_path, "rb") as f: x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) return x509.get_serial_number() diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index 1e85ae808..9443ae266 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -14,8 +14,8 @@ import sys import textwrap from typing import List -import zope.interface import zope.component +import zope.interface from certbot import errors from certbot import interfaces @@ -98,8 +98,7 @@ def input_with_timeout(prompt=None, timeout=36000.0): return line.rstrip('\n') -def notify(msg): - # type: (str) -> None +def notify(msg: str) -> None: """Display a basic status message. :param str msg: message to display @@ -636,8 +635,7 @@ def _parens_around_char(label): return "({first}){rest}".format(first=label[0], rest=label[1:]) -def summarize_domain_list(domains): - # type: (List[str]) -> str +def summarize_domain_list(domains: List[str]) -> str: """Summarizes a list of domains in the format of: example.com.com and N more domains or if there is are only two domains: diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 9a18b32f2..0a842e108 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -59,8 +59,7 @@ class RevocationChecker: else: self.host_args = lambda host: ["Host", host] - def ocsp_revoked(self, cert): - # type: (RenewableCert) -> bool + def ocsp_revoked(self, cert: RenewableCert) -> bool: """Get revoked status for a particular cert version. .. todo:: Make this a non-blocking call @@ -72,8 +71,7 @@ class RevocationChecker: """ return self.ocsp_revoked_by_paths(cert.cert_path, cert.chain_path) - def ocsp_revoked_by_paths(self, cert_path, chain_path, timeout=10): - # type: (str, str, int) -> bool + def ocsp_revoked_by_paths(self, cert_path: str, chain_path: str, timeout: int = 10) -> bool: """Performs the OCSP revocation check :param str cert_path: Certificate filepath @@ -102,8 +100,8 @@ class RevocationChecker: return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url, timeout) return _check_ocsp_cryptography(cert_path, chain_path, url, timeout) - def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url, timeout): - # type: (str, str, str, str, int) -> bool + def _check_ocsp_openssl_bin(self, cert_path: str, chain_path: str, + host: str, url: str, timeout: int) -> bool: # Minimal implementation of proxy selection logic as seen in, e.g., cURL # Some things that won't work, but may well be in use somewhere: # - username and password for proxy authentication @@ -140,8 +138,7 @@ class RevocationChecker: return _translate_ocsp_query(cert_path, output, err) -def _determine_ocsp_server(cert_path): - # type: (str) -> Tuple[Optional[str], Optional[str]] +def _determine_ocsp_server(cert_path: str) -> Tuple[Optional[str], Optional[str]]: """Extract the OCSP server host from a certificate. :param str cert_path: Path to the cert we're checking OCSP for @@ -171,8 +168,7 @@ def _determine_ocsp_server(cert_path): return None, None -def _check_ocsp_cryptography(cert_path, chain_path, url, timeout): - # type: (str, str, str, int) -> bool +def _check_ocsp_cryptography(cert_path: str, chain_path: str, url: str, timeout: int) -> bool: # Retrieve OCSP response with open(chain_path, 'rb') as file_handler: issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index 5e1c29669..06acefc69 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -9,7 +9,7 @@ from josepy import util as jose_util import pkg_resources import zope.interface -from certbot import achallenges # pylint: disable=unused-import +from certbot import achallenges from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -313,8 +313,8 @@ class ChallengePerformer: def __init__(self, configurator): self.configurator = configurator - self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] - self.indices = [] # type: List[int] + self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = [] + self.indices: List[int] = [] def add_chall(self, achall, idx=None): """Store challenge to be performed when perform() is called. diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index 5fec5e0d6..10efa5f85 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -116,8 +116,9 @@ class LexiconClient: return None -def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_options): - # type: (str, Dict, Dict) -> Union[ConfigResolver, Dict] +def build_lexicon_config(lexicon_provider_name: str, + lexicon_options: Dict, provider_options: Dict + ) -> Union[ConfigResolver, Dict]: """ Convenient function to build a Lexicon 2.x/3.x config object. :param str lexicon_provider_name: the name of the lexicon provider to use @@ -126,7 +127,7 @@ def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_option :return: configuration to apply to the provider :rtype: ConfigurationResolver or dict """ - config = {'provider_name': lexicon_provider_name} # type: Dict[str, Any] + config: Dict[str, Any] = {'provider_name': lexicon_provider_name} config.update(lexicon_options) if not ConfigResolver: # Lexicon 2.x diff --git a/certbot/certbot/plugins/enhancements.py b/certbot/certbot/plugins/enhancements.py index 3c3db7e71..c1de4d44f 100644 --- a/certbot/certbot/plugins/enhancements.py +++ b/certbot/certbot/plugins/enhancements.py @@ -156,7 +156,7 @@ class AutoHSTSEnhancement(object, metaclass=abc.ABCMeta): # This is used to configure internal new style enhancements in Certbot. These # enhancement interfaces need to be defined in this file. Please do not modify # this list from plugin code. -_INDEX = [ +_INDEX: List[Dict[str, Any]] = [ { "name": "AutoHSTS", "cli_help": "Gradually increasing max-age value for HTTP Strict Transport "+ @@ -171,4 +171,4 @@ _INDEX = [ "deployer_function": "deploy_autohsts", "enable_function": "enable_autohsts" } -] # type: List[Dict[str, Any]] +] diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index b5dd6a502..0d32d6850 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -42,7 +42,7 @@ class PluginStorage: :raises .errors.PluginStorageError: when unable to open or read the file """ - data = {} # type: Dict[str, Any] + data: Dict[str, Any] = {} filedata = "" try: with open(self._storagepath, 'r') as fh: diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index 0643d5ba7..5f4a08dc7 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -58,7 +58,7 @@ _INITIAL_PID = os.getpid() # the dict are attempted to be cleaned up at program exit. If the # program exits before the lock is cleaned up, it is automatically # released, but the file isn't deleted. -_LOCKS = {} # type: Dict[str, lock.LockFile] +_LOCKS: Dict[str, lock.LockFile] = {} def env_no_snap_for_external_calls(): @@ -216,10 +216,10 @@ def safe_open(path, mode="w", chmod=None): if ``None``. """ - open_args = () # type: Union[Tuple[()], Tuple[int]] + open_args: Union[Tuple[()], Tuple[int]] = () if chmod is not None: open_args = (chmod,) - fdopen_args = () # type: Union[Tuple[()], Tuple[int]] + fdopen_args: Union[Tuple[()], Tuple[int]] = () fd = filesystem.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args) return os.fdopen(fd, mode, *fdopen_args) @@ -577,7 +577,7 @@ def is_wildcard_domain(domain): :rtype: bool """ - wildcard_marker = b"*." # type: Union[Text, bytes] + wildcard_marker: Union[Text, bytes] = b"*." if isinstance(domain, str): wildcard_marker = u"*." return domain.startswith(wildcard_marker) diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 8ae6290bf..4b08fe3a1 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -30,7 +30,7 @@ class CompleterTest(test_util.TempDirTestCase): if self.tempdir[-1] != os.sep: self.tempdir += os.sep - self.paths = [] # type: List[str] + self.paths: List[str] = [] # create some files and directories in temp_dir for c in string.ascii_lowercase: path = os.path.join(self.tempdir, c) diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 9ccd63a3a..265a42023 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -27,7 +27,7 @@ def set_signals(sig_handler_dict): def signal_receiver(signums): """Context manager to catch signals""" signals = [] - prev_handlers = get_signals(signums) # type: Dict[int, Union[int, None, Callable]] + prev_handlers: Dict[int, Union[int, None, Callable]] = get_signals(signums) set_signals({s: lambda s, _: signals.append(s) for s in signums}) yield signals set_signals(prev_handlers) diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 06a641216..b0522bd52 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -267,7 +267,7 @@ class RunSavedPostHooksTest(HookTest): def setUp(self): super(RunSavedPostHooksTest, self).setUp() - self.eventually = [] # type: List[str] + self.eventually: List[str] = [] def test_empty(self): self.assertFalse(self._call_with_mock_execute_and_eventually().called) diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 088faa0fb..33d89c3ea 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -43,7 +43,7 @@ class PreArgParseSetupTest(unittest.TestCase): mock_root_logger.setLevel.assert_called_once_with(logging.DEBUG) self.assertEqual(mock_root_logger.addHandler.call_count, 2) - memory_handler = None # type: Optional[logging.handlers.MemoryHandler] + memory_handler: Optional[logging.handlers.MemoryHandler] = None for call in mock_root_logger.addHandler.call_args_list: handler = call[0][0] if memory_handler is None and isinstance(handler, logging.handlers.MemoryHandler): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 7668a1b6a..d4afed858 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -855,7 +855,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() stdout = io.StringIO() @@ -870,7 +870,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args_unprivileged(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() def throw_error(directory, mode, strict): @@ -892,7 +892,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_init(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() stdout = io.StringIO() @@ -910,7 +910,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_prepare(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() stdout = io.StringIO() diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index a97de24b9..64829c8cc 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -277,7 +277,7 @@ class PluginsRegistryTest(unittest.TestCase): self.plugin_ep.prepare.assert_called_once_with() def test_prepare_order(self): - order = [] # type: List[str] + order: List[str] = [] plugins = dict( (c, mock.MagicMock(prepare=functools.partial(order.append, c))) for c in string.ascii_letters) diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index a5de99e60..027ebdc35 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -52,7 +52,7 @@ class PickPluginTest(unittest.TestCase): self.default = None self.reg = mock.MagicMock() self.question = "Question?" - self.ifaces = [] # type: List[interfaces.IPlugin] + self.ifaces: List[interfaces.IPlugin] = [] def _call(self): from certbot._internal.plugins.selection import pick_plugin diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 596cee622..80347d35a 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -24,9 +24,8 @@ class ServerManagerTest(unittest.TestCase): def setUp(self): from certbot._internal.plugins.standalone import ServerManager - self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] - self.http_01_resources = {} \ - # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] + self.certs: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] = {} + self.http_01_resources: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] = {} self.mgr = ServerManager(self.certs, self.http_01_resources) def test_init(self): From 0962b0fc8346fd948b5a6847fb4f6f0dc90da8a6 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 11 Mar 2021 22:08:20 +0100 Subject: [PATCH 33/39] Kill current snapcraft build when a "Chroot problem" is encountered (#8442) * Kill snapcraft build when a "Chroot problem" is encountered * Display specific helper for "Chroot problem" status and cancel retry mechanism in this case. * Isolate build tmp directories * Configure XDG_CACHE_HOME * Kill snapcraftctl with chroot problem is encountered --- tools/snap/build_remote.py | 66 +++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index e690d2916..69c704e45 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -5,6 +5,8 @@ import glob from multiprocessing import Manager from multiprocessing import Pool from multiprocessing import Process +from multiprocessing.managers import SyncManager +import os from os.path import basename from os.path import dirname from os.path import exists @@ -13,27 +15,46 @@ from os.path import realpath import re import subprocess import sys +import tempfile +from threading import Lock import time +from typing import Dict +from typing import List +from typing import Set +from typing import Tuple CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] -def _execute_build(target, archs, status, workspace): - process = subprocess.Popen([ - 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', - '--build-on', ','.join(archs) - ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, cwd=workspace) +def _execute_build( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + workspace: str) -> Tuple[int, List[str]]: - process_output = [] + with tempfile.TemporaryDirectory() as tempdir: + environ = os.environ.copy() + environ['XDG_CACHE_HOME'] = tempdir + process = subprocess.Popen([ + 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', + '--build-on', ','.join(archs)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True, env=environ, cwd=workspace) + + process_output: List[str] = [] for line in process.stdout: process_output.append(line) _extract_state(target, line, status) + if any(state for state in status[target].values() if state == 'Chroot problem'): + # On this error the snapcraft process stales. Let's finish it. + process.kill() + return process.wait(), process_output -def _build_snap(target, archs, status, running, lock): +def _build_snap( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + running: Dict[str, bool], lock: Lock) -> Dict[str, str]: status[target] = {arch: '...' for arch in archs} if target == 'certbot': @@ -51,7 +72,14 @@ def _build_snap(target, archs, status, running, lock): with lock: dump_output = exit_code != 0 - failed_archs = [arch for arch in archs if status[target][arch] == 'Failed to build'] + failed_archs = [arch for arch in archs if status[target][arch] != 'Successfully built'] + if any(arch for arch in archs if status[target][arch] == 'Chroot problem'): + print('Some builds failed with the status "Chroot problem".') + print('This status is known to make any future build fail until either ' + 'the source code changes or the build on Launchpad is deleted.') + print('Please fix the build appropriately before trying a new one.') + # It is useless to retry in this situation. + retry = 0 if exit_code == 0 and not failed_archs: # We expect to have all target snaps available, or something bad happened. snaps_list = glob.glob(join(workspace, '*.snap')) @@ -80,7 +108,7 @@ def _build_snap(target, archs, status, running, lock): return {target: workspace} -def _extract_state(project, output, status): +def _extract_state(project: str, output: str, status: Dict[str, Dict[str, str]]) -> None: match = re.match(r'^.*arch=(\w+)\s+state=([\w ]+).*$', output) if match: arch = match.group(1) @@ -95,7 +123,7 @@ def _extract_state(project, output, status): status[project] = state -def _dump_status_helper(archs, status): +def _dump_status_helper(archs: Set[str], status: Dict[str, Dict[str, str]]) -> None: headers = ['project', *archs] print(''.join(f'| {item:<25}' for item in headers)) print(f'|{"-" * 26}' * len(headers)) @@ -107,14 +135,18 @@ def _dump_status_helper(archs, status): sys.stdout.flush() -def _dump_status(archs, status, running): +def _dump_status( + archs: Set[str], status: Dict[str, Dict[str, str]], + running: Dict[str, bool]) -> None: while any(running.values()): print(f'Remote build status at {datetime.datetime.now()}') _dump_status_helper(archs, status) time.sleep(10) -def _dump_results(targets, archs, status, workspaces): +def _dump_results( + targets: Set[str], archs: Set[str], status: Dict[str, Dict[str, str]], + workspaces: Dict[str, str]) -> bool: failures = False for target in targets: for arch in archs: @@ -152,8 +184,8 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('targets', nargs='+', choices=['ALL', 'DNS_PLUGINS', 'certbot', *PLUGINS], help='the list of snaps to build') - parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default=['amd64'], - help='the architectures for which snaps are built') + parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], + default=['amd64'], help='the architectures for which snaps are built') parser.add_argument('--timeout', type=int, default=None, help='build process will fail after the provided timeout (in seconds)') args = parser.parse_args() @@ -180,8 +212,10 @@ def main(): print(f' - projects: {", ".join(sorted(targets))}') print() - with Manager() as manager, Pool(processes=len(targets)) as pool: - status = manager.dict() + manager: SyncManager = Manager() + pool = Pool(processes=len(targets)) + with manager, pool: + status: Dict[str, Dict[str, str]] = manager.dict() running = manager.dict({target: True for target in targets}) lock = manager.Lock() From bc892e04c4e05b1b57f59179b7b465f7f981a176 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 11 Mar 2021 22:17:41 +0100 Subject: [PATCH 34/39] Fixing imports in cli module (#8708) While working on #8640, I realized that there were some hidden circular dependencies in certbot._internal.cli package. Then cerbot could break if the order of these imports changes. This PR fixes that and apply isort on top of the result. --- certbot/certbot/_internal/cli/__init__.py | 86 ++++++++----------- certbot/certbot/_internal/cli/group_adder.py | 2 +- certbot/certbot/_internal/cli/helpful.py | 33 +++---- certbot/certbot/_internal/cli/paths_parser.py | 6 +- .../certbot/_internal/cli/plugins_parsing.py | 2 +- certbot/certbot/_internal/cli/subparsers.py | 13 ++- certbot/certbot/_internal/cli/verb_help.py | 6 +- 7 files changed, 61 insertions(+), 87 deletions(-) diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index b64ab0e9c..d48fd419a 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -1,65 +1,51 @@ """Certbot command line argument & config processing.""" # pylint: disable=too-many-lines +import argparse import logging import logging.handlers -import argparse import sys from typing import Optional -import certbot._internal.plugins.selection as plugin_selection -from certbot._internal.plugins import disco as plugins_disco - -# pylint: disable=ungrouped-imports import certbot from certbot._internal import constants - -import certbot.plugins.enhancements as enhancements - - -from certbot._internal.cli.cli_constants import ( - LEAUTO, - old_path_fragment, - new_path_prefix, - cli_command, - SHORT_USAGE, - COMMAND_OVERVIEW, - HELP_AND_VERSION_USAGE, - ARGPARSE_PARAMS_TO_REMOVE, - EXIT_ACTIONS, - ZERO_ARG_ACTIONS, - VAR_MODIFIERS, - DEPRECATED_OPTIONS -) - -from certbot._internal.cli.cli_utils import ( - _Default, - read_file, - flag_default, - config_help, - HelpfulArgumentGroup, - CustomHelpFormatter, - _DomainsAction, - add_domains, - CaseInsensitiveList, - _user_agent_comment_type, - _EncodeReasonAction, - parse_preferred_challenges, - _PrefChallAction, - _DeployHookAction, - _RenewHookAction, - nonnegative_int -) - -# These imports depend on cli_constants and cli_utils. -from certbot._internal.cli.verb_help import VERB_HELP, VERB_HELP_MAP +from certbot._internal.cli.cli_constants import ARGPARSE_PARAMS_TO_REMOVE +from certbot._internal.cli.cli_constants import cli_command +from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW +from certbot._internal.cli.cli_constants import DEPRECATED_OPTIONS +from certbot._internal.cli.cli_constants import EXIT_ACTIONS +from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE +from certbot._internal.cli.cli_constants import LEAUTO +from certbot._internal.cli.cli_constants import new_path_prefix +from certbot._internal.cli.cli_constants import old_path_fragment +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_constants import VAR_MODIFIERS +from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS +from certbot._internal.cli.cli_utils import _Default +from certbot._internal.cli.cli_utils import _DeployHookAction +from certbot._internal.cli.cli_utils import _DomainsAction +from certbot._internal.cli.cli_utils import _EncodeReasonAction +from certbot._internal.cli.cli_utils import _PrefChallAction +from certbot._internal.cli.cli_utils import _RenewHookAction +from certbot._internal.cli.cli_utils import _user_agent_comment_type +from certbot._internal.cli.cli_utils import add_domains +from certbot._internal.cli.cli_utils import CaseInsensitiveList +from certbot._internal.cli.cli_utils import config_help +from certbot._internal.cli.cli_utils import CustomHelpFormatter +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import HelpfulArgumentGroup +from certbot._internal.cli.cli_utils import nonnegative_int +from certbot._internal.cli.cli_utils import parse_preferred_challenges +from certbot._internal.cli.cli_utils import read_file from certbot._internal.cli.group_adder import _add_all_groups -from certbot._internal.cli.subparsers import _create_subparsers +from certbot._internal.cli.helpful import HelpfulArgumentParser from certbot._internal.cli.paths_parser import _paths_parser from certbot._internal.cli.plugins_parsing import _plugins_parsing - -# These imports depend on some or all of the submodules for cli. -from certbot._internal.cli.helpful import HelpfulArgumentParser -# pylint: enable=ungrouped-imports +from certbot._internal.cli.subparsers import _create_subparsers +from certbot._internal.cli.verb_help import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP_MAP +from certbot._internal.plugins import disco as plugins_disco +import certbot._internal.plugins.selection as plugin_selection +import certbot.plugins.enhancements as enhancements logger = logging.getLogger(__name__) diff --git a/certbot/certbot/_internal/cli/group_adder.py b/certbot/certbot/_internal/cli/group_adder.py index f22fbc496..0c54c9fe1 100644 --- a/certbot/certbot/_internal/cli/group_adder.py +++ b/certbot/certbot/_internal/cli/group_adder.py @@ -1,6 +1,6 @@ """This module contains a function to add the groups of arguments for the help display""" -from certbot._internal.cli import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP def _add_all_groups(helpful): diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index d36147089..f185bdc26 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -11,35 +11,30 @@ from typing import Dict import configargparse import zope.component import zope.interface - from zope.interface import interfaces as zope_interfaces from certbot import crypto_util from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import os from certbot._internal import constants from certbot._internal import hooks - +from certbot._internal.cli.cli_constants import ARGPARSE_PARAMS_TO_REMOVE +from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW +from certbot._internal.cli.cli_constants import EXIT_ACTIONS +from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS +from certbot._internal.cli.cli_utils import _Default +from certbot._internal.cli.cli_utils import add_domains +from certbot._internal.cli.cli_utils import CustomHelpFormatter +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import HelpfulArgumentGroup +from certbot._internal.cli.verb_help import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP_MAP +from certbot.compat import os from certbot.display import util as display_util -from certbot._internal.cli import ( - SHORT_USAGE, - CustomHelpFormatter, - flag_default, - VERB_HELP, - VERB_HELP_MAP, - COMMAND_OVERVIEW, - HELP_AND_VERSION_USAGE, - _Default, - add_domains, - EXIT_ACTIONS, - ZERO_ARG_ACTIONS, - ARGPARSE_PARAMS_TO_REMOVE, - HelpfulArgumentGroup -) - class HelpfulArgumentParser: """Argparse Wrapper. diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py index 43ee41c88..6197a4bf9 100644 --- a/certbot/certbot/_internal/cli/paths_parser.py +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -1,10 +1,8 @@ """This is a module that adds configuration to the argument parser regarding paths for certificates""" +from certbot._internal.cli.cli_utils import config_help +from certbot._internal.cli.cli_utils import flag_default from certbot.compat import os -from certbot._internal.cli import ( - flag_default, - config_help -) def _paths_parser(helpful): diff --git a/certbot/certbot/_internal/cli/plugins_parsing.py b/certbot/certbot/_internal/cli/plugins_parsing.py index 9e11ad3ab..bbfdf22da 100644 --- a/certbot/certbot/_internal/cli/plugins_parsing.py +++ b/certbot/certbot/_internal/cli/plugins_parsing.py @@ -1,5 +1,5 @@ """This is a module that handles parsing of plugins for the argument parser""" -from certbot._internal.cli import flag_default +from certbot._internal.cli.cli_utils import flag_default def _plugins_parsing(helpful, plugins): diff --git a/certbot/certbot/_internal/cli/subparsers.py b/certbot/certbot/_internal/cli/subparsers.py index 13f8705ce..822381d21 100644 --- a/certbot/certbot/_internal/cli/subparsers.py +++ b/certbot/certbot/_internal/cli/subparsers.py @@ -1,14 +1,11 @@ """This module creates subparsers for the argument parser""" from certbot import interfaces from certbot._internal import constants - -from certbot._internal.cli import ( - flag_default, - read_file, - CaseInsensitiveList, - _user_agent_comment_type, - _EncodeReasonAction -) +from certbot._internal.cli.cli_utils import _EncodeReasonAction +from certbot._internal.cli.cli_utils import _user_agent_comment_type +from certbot._internal.cli.cli_utils import CaseInsensitiveList +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import read_file def _create_subparsers(helpful): diff --git a/certbot/certbot/_internal/cli/verb_help.py b/certbot/certbot/_internal/cli/verb_help.py index 131cfec96..dfb4e9c67 100644 --- a/certbot/certbot/_internal/cli/verb_help.py +++ b/certbot/certbot/_internal/cli/verb_help.py @@ -1,9 +1,7 @@ """This module contain help information for verbs supported by certbot""" +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_utils import flag_default from certbot.compat import os -from certbot._internal.cli import ( - SHORT_USAGE, - flag_default -) # The attributes here are: # short: a string that will be displayed by "certbot -h commands" From 2324c1bb7ad977bc3525ebef1a3f61410e0e245f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 15 Mar 2021 14:10:44 -0700 Subject: [PATCH 35/39] Update installing from source instructions (#8713) --- certbot/docs/install.rst | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index 8ae1c82f2..ac6c798eb 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -270,15 +270,10 @@ Disable and remove the swapfile once the virtual environment is constructed:: user@webserver:~$ sudo swapoff /tmp/swapfile user@webserver:~$ sudo rm /tmp/swapfile -Installing from source ----------------------- +Pip +--- -Installation from source is only supported for developers and the -whole process is described in the :doc:`contributing`. - -.. warning:: Please do **not** use ``python certbot/setup.py install``, ``python pip - install certbot``, or ``easy_install certbot``. Please do **not** attempt the - installation commands as superuser/root and/or without virtual environment, - e.g. ``sudo python certbot/setup.py install``, ``sudo pip install``, ``sudo - ./venv/bin/...``. These modes of operation might corrupt your operating - system and are **not supported** by the Certbot team! +Installing Certbot through pip is only supported on a best effort basis and +when using a virtual environment. Instructions for installing Certbot through +pip can be found at https://certbot.eff.org/instructions by selecting your +server software and then choosing "pip" in the "System" dropdown menu. From 1b39d3dc478ee8df1e39ec6c8ed1e44edcca7409 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 Mar 2021 17:53:43 -0700 Subject: [PATCH 36/39] switch to wait_until_running (#8715) --- tests/letstest/multitester.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index f079f893f..2e4399000 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -190,18 +190,16 @@ def block_until_ssh_open(ipstring, wait_time=10, timeout=120): t_elapsed += wait_time sock.close() -def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20): +def block_until_instance_ready(booting_instance, extra_wait_time=20): "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections" - state = booting_instance.state['Name'] - ip = booting_instance.public_ip_address - while state != 'running' or ip is None: - time.sleep(wait_time) - # The instance needs to be reloaded to update its local attributes. See - # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Instance.reload. - booting_instance.reload() - state = booting_instance.state['Name'] - ip = booting_instance.public_ip_address - block_until_ssh_open(ip) + booting_instance.wait_until_running() + # The instance needs to be reloaded to update its local attributes. See + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Instance.reload. + booting_instance.reload() + # After waiting for the instance to be running and reloading the instance + # state, we should have an IP address. + assert booting_instance.public_ip_address is not None + block_until_ssh_open(booting_instance.public_ip_address) time.sleep(extra_wait_time) return booting_instance From 40ae5d939ee0449ce54b8bf12e75e9311e361d91 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 19 Mar 2021 14:27:29 -0700 Subject: [PATCH 37/39] Fix linux-py39-cover test (#8720) * update setuptools * upgrade Markupsafe --- tools/dev_constraints.txt | 2 +- tools/pipstrap_constraints.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index f4059a3f9..4f5eda34e 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -56,7 +56,7 @@ jsonschema==2.6.0 lazy-object-proxy==1.4.3 logger==1.4 logilab-common==1.4.1 -MarkupSafe==1.0 +MarkupSafe==1.1.1 mccabe==0.6.1 more-itertools==5.0.0 msrest==0.6.18 diff --git a/tools/pipstrap_constraints.txt b/tools/pipstrap_constraints.txt index 5de9e147d..54ab8b429 100644 --- a/tools/pipstrap_constraints.txt +++ b/tools/pipstrap_constraints.txt @@ -10,9 +10,9 @@ pip==20.2.4 \ --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \ --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1 -setuptools==44.1.1 \ - --hash=sha256:27a714c09253134e60a6fa68130f78c7037e5562c4f21f8f318f2ae900d152d5 \ - --hash=sha256:c67aa55db532a0dadc4d2e20ba9961cbd3ccc84d544e9029699822542b5a476b +setuptools==54.1.2 \ + --hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b \ + --hash=sha256:ebd0148faf627b569c8d2a1b20f5d3b09c873f12739d71c7ee88f037d5be82ff wheel==0.35.1 \ --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \ --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f From 6bc8b3d2ba9a0f22a6682a46ae4905c462e19388 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sun, 21 Mar 2021 22:39:54 +0100 Subject: [PATCH 38/39] Precise the certificate naming convention mechanism in the compatibility document (#8652) * Precise the certificate naming convention mechanism in a note. * Add certificate name convention in user guide, refer to it in compatibility page. * Update certbot/docs/compatibility.rst Co-authored-by: alexzorin * Update certbot/docs/using.rst Co-authored-by: alexzorin * Update certbot/docs/using.rst Co-authored-by: alexzorin * Improve the note about naming conventions Co-authored-by: alexzorin --- certbot/docs/compatibility.rst | 6 +++--- certbot/docs/using.rst | 24 ++++++++++++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/certbot/docs/compatibility.rst b/certbot/docs/compatibility.rst index a4f33c281..d94642ec6 100644 --- a/certbot/docs/compatibility.rst +++ b/certbot/docs/compatibility.rst @@ -21,9 +21,9 @@ may change at any time. The second is that Certbot's behavior should only be considered stable with certain files but not all. Files with which users should expect Certbot to maintain its current behavior with are: -* ``/etc/letsencrypt/live//{cert,chain,fullchain,privkey}.pem`` where - ```` is the name given to ``--cert-name``. If ``--cert-name`` is not - set by the user, it is the first domain given to ``--domains``. +* ``/etc/letsencrypt/live/$domain/{cert,chain,fullchain,privkey}.pem``, where + ``$domain`` is the certificate name (see :ref:`where-certs` + for more details) * :ref:`CLI configuration files ` * Hook directories in ``/etc/letsencrypt/renewal-hooks`` diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 1d97caecc..87c4b9569 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -717,12 +717,24 @@ Where are my certificates? ========================== All generated keys and issued certificates can be found in -``/etc/letsencrypt/live/$domain``. In the case of creating a SAN certificate -with multiple alternative names, ``$domain`` is the first domain passed in -via -d parameter. Rather than copying, please point -your (web) server configuration directly to those files (or create -symlinks). During the renewal_, ``/etc/letsencrypt/live`` is updated -with the latest necessary files. +``/etc/letsencrypt/live/$domain``, where ``$domain`` is the certificate +name (see the note below). Rather than copying, please point your (web) +server configuration directly to those files (or create symlinks). +During the renewal_, ``/etc/letsencrypt/live`` is updated with the latest +necessary files. + +.. note:: + The certificate name ``$domain`` used in the path ``/etc/letsencrypt/live/$domain`` + follows this convention: + + * it is the name given to ``--cert-name``, + * if ``--cert-name`` is not set by the user it is the first domain given to + ``--domains``, + * if the first domain is a wildcard domain (eg. ``*.example.com``) the + certificate name will be ``example.com``, + * if a name collision would occur with a certificate already named ``example.com``, + the new certificate name will be constructed using a numerical sequence + as ``example.com-001``. For historical reasons, the containing directories are created with permissions of ``0700`` meaning that certificates are accessible only From ae2247163e42934c0486c36079bc7ae50bc4d4e2 Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Sun, 21 Mar 2021 22:42:23 +0100 Subject: [PATCH 39/39] Remove empty lines from `certbot certificates` when (#8723) .. envoked with `--cert-name` or `-d`. --- certbot/CHANGELOG.md | 3 ++- certbot/certbot/_internal/cert_manager.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 62bceb74d..9b37e9a1a 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -17,7 +17,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* +* Don't output an empty line for a hidden certificate when `certbot certificates` is being used + in combination with `--cert-name` or `-d`. More details about these changes can be found on our GitHub repo. diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index 8fab5735a..b9b4ad2d7 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -266,9 +266,9 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False): checker = ocsp.RevocationChecker() if config.certname and cert.lineagename != config.certname and not skip_filter_checks: - return "" + return None if config.domains and not set(config.domains).issubset(cert.names()): - return "" + return None now = pytz.UTC.fromutc(datetime.datetime.utcnow()) reasons = [] @@ -358,7 +358,9 @@ def _report_human_readable(config, parsed_certs): """Format a results report for a parsed cert""" certinfo = [] for cert in parsed_certs: - certinfo.append(human_readable_cert_info(config, cert)) + cert_info = human_readable_cert_info(config, cert) + if cert_info is not None: + certinfo.append(cert_info) return "\n".join(certinfo)