diff --git a/.pylintrc b/.pylintrc index 0e78828bd..a2468b0cf 100644 --- a/.pylintrc +++ b/.pylintrc @@ -254,7 +254,7 @@ ignore-mixin-members=yes # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis -ignored-modules=pkg_resources,confargparse,argparse,six.moves,six.moves.urllib +ignored-modules=pkg_resources,confargparse,argparse # import errors ignored only in 1.4.4 # https://bitbucket.org/logilab/pylint/commits/cd000904c9e2 diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 376e9a382..82f7ae1b8 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -8,15 +8,15 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose -import requests -import six -from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from OpenSSL import crypto +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 +import requests from acme import crypto_util from acme import errors from acme import fields -from acme.mixins import ResourceMixin, TypeMixin +from acme.mixins import ResourceMixin +from acme.mixins import TypeMixin logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) class Challenge(jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge.""" - TYPES = {} # type: dict + TYPES: dict = {} @classmethod def from_json(cls, jobj): @@ -38,7 +38,7 @@ class Challenge(jose.TypedJSONObjectWithFields): class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge response.""" - TYPES = {} # type: dict + TYPES: dict = {} resource_type = 'challenge' resource = fields.Resource(resource_type) @@ -145,8 +145,7 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return jobj -@six.add_metaclass(abc.ABCMeta) -class KeyAuthorizationChallenge(_TokenChallenge): +class KeyAuthorizationChallenge(_TokenChallenge, metaclass=abc.ABCMeta): """Challenge based on Key Authorization. :param response_cls: Subclass of `KeyAuthorizationChallengeResponse` diff --git a/acme/acme/client.py b/acme/acme/client.py index 6adfe4b78..f5aa1ff9e 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -4,9 +4,14 @@ import collections import datetime from email.utils import parsedate_tz import heapq +import http.client as http_client import logging import re import time +from typing import Dict +from typing import List +from typing import Set +from typing import Text import josepy as jose import OpenSSL @@ -14,17 +19,11 @@ import requests from requests.adapters import HTTPAdapter from requests.utils import parse_header_links from requests_toolbelt.adapters.source import SourceAddressAdapter -import six -from six.moves import http_client from acme import crypto_util from acme import errors from acme import jws from acme import messages -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Text from acme.mixins import VersionedLEACMEMixin logger = logging.getLogger(__name__) @@ -34,7 +33,7 @@ DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class ClientBase(object): +class ClientBase: """ACME client base object. :ivar messages.Directory directory: @@ -113,8 +112,9 @@ class ClientBase(object): """ return self.update_registration(regr, update={'status': 'deactivated'}) - def deactivate_authorization(self, authzr): - # type: (messages.AuthorizationResource) -> messages.AuthorizationResource + def deactivate_authorization(self, + authzr: messages.AuthorizationResource + ) -> messages.AuthorizationResource: """Deactivate authorization. :param messages.AuthorizationResource authzr: The Authorization resource @@ -248,7 +248,7 @@ class Client(ClientBase): if net is None: net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) - if isinstance(directory, six.string_types): + if isinstance(directory, str): directory = messages.Directory.from_json( net.get(directory).json()) super(Client, self).__init__(directory=directory, @@ -424,7 +424,7 @@ class Client(ClientBase): """ assert max_attempts > 0 - attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int] + attempts: Dict[messages.AuthorizationResource, int] = collections.defaultdict(int) exhausted = set() # priority queue with datetime.datetime (based on Retry-After) as key, @@ -463,7 +463,7 @@ class Client(ClientBase): exhausted.add(authzr) if exhausted or any(authzr.body.status == messages.STATUS_INVALID - for authzr in six.itervalues(updated)): + for authzr in updated.values()): raise errors.PollError(exhausted, updated) updated_authzrs = tuple(updated[authzr] for authzr in authzrs) @@ -537,7 +537,7 @@ class Client(ClientBase): :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509` """ - chain = [] # type: List[jose.ComparableX509] + chain: List[jose.ComparableX509] = [] uri = certr.cert_chain_uri while uri is not None and len(chain) < max_length: response, cert = self._get_cert(uri) @@ -796,7 +796,7 @@ class ClientV2(ClientBase): if 'rel' in l and 'url' in l and l['rel'] == relation_type] -class BackwardsCompatibleClientV2(object): +class BackwardsCompatibleClientV2: """ACME client wrapper that tends towards V2-style calls, but supports V1 servers. @@ -939,7 +939,7 @@ class BackwardsCompatibleClientV2(object): return self.client.external_account_required() -class ClientNetwork(object): +class ClientNetwork: """Wrapper around requests that signs POSTs for authentication. Also adds user agent, and handles Content-Type. @@ -969,7 +969,7 @@ class ClientNetwork(object): self.account = account self.alg = alg self.verify_ssl = verify_ssl - self._nonces = set() # type: Set[Text] + self._nonces: Set[Text] = set() self.user_agent = user_agent self.session = requests.Session() self._default_timeout = timeout diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 4b58db328..749478bf5 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -5,15 +5,15 @@ import logging import os import re import socket +from typing import Callable +from typing import Tuple +from typing import Union import josepy as jose from OpenSSL import crypto from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from acme import errors -from acme.magic_typing import Callable -from acme.magic_typing import Tuple -from acme.magic_typing import Union logger = logging.getLogger(__name__) @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore -class _DefaultCertSelection(object): +class _DefaultCertSelection: def __init__(self, certs): self.certs = certs @@ -36,7 +36,7 @@ class _DefaultCertSelection(object): return self.certs.get(server_name, None) -class SSLSocket(object): # pylint: disable=too-few-public-methods +class SSLSocket: # pylint: disable=too-few-public-methods """SSL wrapper for sockets. :ivar socket sock: Original wrapped socket. @@ -93,7 +93,7 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) - class FakeConnection(object): + class FakeConnection: """Fake OpenSSL.SSL.Connection.""" # pylint: disable=missing-function-docstring @@ -168,7 +168,7 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu source_address[1] ) if any(source_address) else "" ) - socket_tuple = (host, port) # type: Tuple[str, int] + socket_tuple: Tuple[str, int] = (host, port) sock = socket.create_connection(socket_tuple, **socket_kwargs) # type: ignore except socket.error as error: raise errors.Error(error) @@ -256,7 +256,7 @@ def _pyopenssl_cert_or_req_san(cert_or_req): if isinstance(cert_or_req, crypto.X509): # pylint: disable=line-too-long - func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] + func: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] = crypto.dump_certificate else: func = crypto.dump_certificate_request text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 388fc4a58..8190fa552 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -1,16 +1,17 @@ -"""Shim class to not have to depend on typing module in prod.""" -import sys +"""Simple shim around the typing module. +This was useful when this code supported Python 2 and typing wasn't always +available. This code is being kept for now for backwards compatibility. -class TypingClass(object): +""" +import warnings +from typing import * # pylint: disable=wildcard-import, unused-wildcard-import +from typing import Collection, IO # type: ignore + +warnings.warn("acme.magic_typing is deprecated and will be removed in a future release.", + DeprecationWarning) + +class TypingClass: """Ignore import errors by getting anything""" def __getattr__(self, name): - return None - -try: - # mypy doesn't respect modifying sys.modules - from typing import * # pylint: disable=wildcard-import, unused-wildcard-import - from typing import Collection, IO # type: ignore -except ImportError: - # mypy complains because TypingClass is not a module - sys.modules[__name__] = TypingClass() # type: ignore + return None # pragma: no cover diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 3a505843d..5e7e22c34 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,8 +1,8 @@ """ACME protocol messages.""" import json +from collections.abc import Hashable import josepy as jose -import six from acme import challenges from acme import errors @@ -11,13 +11,6 @@ from acme import jws from acme import util from acme.mixins import ResourceMixin -try: - from collections.abc import Hashable -except ImportError: # pragma: no cover - from collections import Hashable - - - OLD_ERROR_PREFIX = "urn:acme:error:" ERROR_PREFIX = "urn:ietf:params:acme:error:" @@ -68,7 +61,6 @@ def is_acme_error(err): return False -@six.python_2_unicode_compatible class Error(jose.JSONObjectWithFields, errors.Error): """ACME error. @@ -158,13 +150,10 @@ class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore def __hash__(self): return hash((self.__class__, self.name)) - def __ne__(self, other): - return not self == other - class Status(_Constant): """ACME "status" field.""" - POSSIBLE_NAMES = {} # type: dict + POSSIBLE_NAMES: dict = {} STATUS_UNKNOWN = Status('unknown') STATUS_PENDING = Status('pending') STATUS_PROCESSING = Status('processing') @@ -177,7 +166,7 @@ STATUS_DEACTIVATED = Status('deactivated') class IdentifierType(_Constant): """ACME identifier type.""" - POSSIBLE_NAMES = {} # type: dict + POSSIBLE_NAMES: dict = {} IDENTIFIER_FQDN = IdentifierType('dns') # IdentifierDNS in Boulder @@ -195,7 +184,7 @@ class Identifier(jose.JSONObjectWithFields): class Directory(jose.JSONDeSerializable): """Directory.""" - _REGISTERED_TYPES = {} # type: dict + _REGISTERED_TYPES: dict = {} class Meta(jose.JSONObjectWithFields): """Directory Meta.""" @@ -285,7 +274,7 @@ class ResourceBody(jose.JSONObjectWithFields): """ACME Resource Body.""" -class ExternalAccountBinding(object): +class ExternalAccountBinding: """ACME External Account Binding""" @classmethod diff --git a/acme/acme/mixins.py b/acme/acme/mixins.py index 1cd050ccc..0e1e0977c 100644 --- a/acme/acme/mixins.py +++ b/acme/acme/mixins.py @@ -1,7 +1,7 @@ """Useful mixins for Challenge and Resource objects""" -class VersionedLEACMEMixin(object): +class VersionedLEACMEMixin: """This mixin stores the version of Let's Encrypt's endpoint being used.""" @property def le_acme_version(self): diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 7a61ba868..0b5a8b5c7 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -1,17 +1,16 @@ """Support for standalone client challenge solvers. """ import collections import functools +import http.client as http_client +import http.server as BaseHTTPServer import logging import socket +import socketserver import threading - -from six.moves import BaseHTTPServer # type: ignore -from six.moves import http_client -from six.moves import socketserver # type: ignore +from typing import List from acme import challenges from acme import crypto_util -from acme.magic_typing import List logger = logging.getLogger(__name__) @@ -54,7 +53,7 @@ class ACMEServerMixin: allow_reuse_address = True -class BaseDualNetworkedServers(object): +class BaseDualNetworkedServers: """Base class for a pair of IPv6 and IPv4 servers that tries to do everything it's asked for both servers, but where failures in one server don't affect the other. @@ -64,8 +63,8 @@ class BaseDualNetworkedServers(object): def __init__(self, ServerClass, server_address, *remaining_args, **kwargs): port = server_address[1] - self.threads = [] # type: List[threading.Thread] - self.servers = [] # type: List[ACMEServerMixin] + self.threads: List[threading.Thread] = [] + self.servers: List[ACMEServerMixin] = [] # Must try True first. # Ubuntu, for example, will fail to bind to IPv4 if we've already bound diff --git a/acme/acme/util.py b/acme/acme/util.py index b3b0d79eb..20fa455fd 100644 --- a/acme/acme/util.py +++ b/acme/acme/util.py @@ -1,7 +1,6 @@ """ACME utilities.""" -import six def map_keys(dikt, func): """Map dictionary keys.""" - return {func(key): value for key, value in six.iteritems(dikt)} + return {func(key): value for key, value in dikt.items()} diff --git a/acme/docs/conf.py b/acme/docs/conf.py index 9d3c4d05c..d419326df 100644 --- a/acme/docs/conf.py +++ b/acme/docs/conf.py @@ -87,7 +87,6 @@ language = 'en' # directories to ignore when looking for source files. exclude_patterns = [ '_build', - 'man/*' ] # The reST default role (used for this markup: `text`) to use for all diff --git a/acme/docs/man/jws.rst b/acme/docs/man/jws.rst index d7ff8f405..7e83328b2 100644 --- a/acme/docs/man/jws.rst +++ b/acme/docs/man/jws.rst @@ -1 +1,3 @@ +:orphan: + .. literalinclude:: ../jws-help.txt diff --git a/acme/setup.py b/acme/setup.py index 745169cbf..90ab5c9f3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -18,7 +18,6 @@ install_requires = [ 'requests>=2.6.0', 'requests-toolbelt>=0.3.0', 'setuptools>=39.0.1', - 'six>=1.11.0', ] dev_extras = [ diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index 22e67be3c..cc604b0de 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -1,11 +1,11 @@ """Tests for acme.challenges.""" +import urllib.parse as urllib_parse import unittest from unittest import mock import josepy as jose import OpenSSL import requests -from six.moves.urllib import parse as urllib_parse from acme import errors diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index a1be59056..14247335c 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -2,6 +2,7 @@ # pylint: disable=too-many-lines import copy import datetime +import http.client as http_client import json import unittest from unittest import mock @@ -9,7 +10,6 @@ from unittest import mock import josepy as jose import OpenSSL import requests -from six.moves import http_client # pylint: disable=import-error from acme import challenges from acme import errors @@ -61,7 +61,7 @@ class ClientTestBase(unittest.TestCase): self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212') reg = messages.Registration( contact=self.contact, key=KEY.public_key()) - the_arg = dict(reg) # type: Dict + the_arg: Dict = dict(reg) self.new_reg = messages.NewRegistration(**the_arg) self.regr = messages.RegistrationResource( body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index 705a3c856..8075b68ed 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -1,14 +1,13 @@ """Tests for acme.crypto_util.""" import itertools import socket +import socketserver import threading import time import unittest import josepy as jose import OpenSSL -import six -from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import errors import test_util @@ -27,8 +26,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): class _TestServer(socketserver.TCPServer): - # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init - def server_bind(self): # pylint: disable=missing-docstring self.socket = SSLSocket(socket.socket(), certs) @@ -62,7 +59,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): self.assertRaises(errors.Error, self._probe, b'bar') def test_probe_connection_error(self): - # pylint has a hard time with six self.server.server_close() original_timeout = socket.getdefaulttimeout() try: @@ -121,9 +117,9 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): @classmethod def _get_idn_names(cls): """Returns expected names from '{cert,csr}-idnsans.pem'.""" - chars = [six.unichr(i) for i in itertools.chain(range(0x3c3, 0x400), - range(0x641, 0x6fc), - range(0x1820, 0x1877))] + chars = [chr(i) for i in itertools.chain(range(0x3c3, 0x400), + range(0x641, 0x6fc), + range(0x1820, 0x1877))] return [''.join(chars[i: i + 45]) + '.invalid' for i in range(0, len(chars), 45)] @@ -184,7 +180,7 @@ class RandomSnTest(unittest.TestCase): def setUp(self): self.cert_count = 5 - self.serial_num = [] # type: List[int] + self.serial_num: List[int] = [] self.key = OpenSSL.crypto.PKey() self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 048995916..d470337bd 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -1,6 +1,7 @@ """Tests for acme.magic_typing.""" import sys import unittest +import warnings from unittest import mock @@ -9,32 +10,21 @@ class MagicTypingTest(unittest.TestCase): def test_import_success(self): try: import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover typing_class_mock = mock.MagicMock() text_mock = mock.MagicMock() typing_class_mock.Text = text_mock sys.modules['typing'] = typing_class_mock if 'acme.magic_typing' in sys.modules: - del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text + del sys.modules['acme.magic_typing'] # pragma: no cover + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + from acme.magic_typing import Text self.assertEqual(Text, text_mock) del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing - def test_import_failure(self): - try: - import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover - sys.modules['typing'] = None - if 'acme.magic_typing' in sys.modules: - del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text - self.assertTrue(Text is None) - del sys.modules['acme.magic_typing'] - sys.modules['typing'] = temp_typing - if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index 74d1737ec..99a3a9ce4 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -1,4 +1,5 @@ """Tests for acme.messages.""" +from typing import Dict import unittest from unittest import mock @@ -81,7 +82,7 @@ class ConstantTest(unittest.TestCase): from acme.messages import _Constant class MockConstant(_Constant): # pylint: disable=missing-docstring - POSSIBLE_NAMES = {} # type: Dict + POSSIBLE_NAMES: Dict = {} self.MockConstant = MockConstant # pylint: disable=invalid-name self.const_a = MockConstant('a') diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 5bbc2ccce..a02d008a0 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -1,13 +1,13 @@ """Tests for acme.standalone.""" +import http.client as http_client import socket +import socketserver import threading import unittest from unittest import mock import josepy as jose import requests -from six.moves import http_client # pylint: disable=import-error -from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import challenges from acme import crypto_util @@ -41,7 +41,7 @@ class HTTP01ServerTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() # type: Set + self.resources: Set = set() from acme.standalone import HTTP01Server self.server = HTTP01Server(('', 0), resources=self.resources) @@ -218,7 +218,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() # type: Set + self.resources: Set = set() from acme.standalone import HTTP01DualNetworkedServers self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources) diff --git a/certbot-apache/certbot_apache/_internal/apache_util.py b/certbot-apache/certbot_apache/_internal/apache_util.py index 93612f424..9b9855a45 100644 --- a/certbot-apache/certbot_apache/_internal/apache_util.py +++ b/certbot-apache/certbot_apache/_internal/apache_util.py @@ -9,7 +9,6 @@ import pkg_resources from certbot import errors from certbot import util - from certbot.compat import os logger = logging.getLogger(__name__) diff --git a/certbot-apache/certbot_apache/_internal/assertions.py b/certbot-apache/certbot_apache/_internal/assertions.py index 2b2ce4f50..53603c526 100644 --- a/certbot-apache/certbot_apache/_internal/assertions.py +++ b/certbot-apache/certbot_apache/_internal/assertions.py @@ -3,7 +3,6 @@ import fnmatch from certbot_apache._internal import interfaces - PASS = "CERTBOT_PASS_ASSERT" diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index 3b2ce40d8..4549e935a 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -64,10 +64,10 @@ Translates over to: "/files/etc/apache2/apache2.conf/bLoCk[1]", ] """ -from acme.magic_typing import Set +from typing import Set + from certbot import errors from certbot.compat import os - from certbot_apache._internal import apache_util from certbot_apache._internal import assertions from certbot_apache._internal import interfaces @@ -355,7 +355,7 @@ class AugeasBlockNode(AugeasDirectiveNode): ownpath = self.metadata.get("augeaspath") directives = self.parser.find_dir(name, start=ownpath, exclude=exclude) - already_parsed = set() # type: Set[str] + already_parsed: Set[str] = set() for directive in directives: # Remove the /arg part from the Augeas path directive = directive.partition("/arg")[0] diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 16def1998..29648d4c1 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -1,28 +1,23 @@ """Apache Configurator.""" # pylint: disable=too-many-lines from collections import defaultdict -from distutils.version import LooseVersion import copy +from distutils.version import LooseVersion import fnmatch import logging import re import socket import time +from typing import DefaultDict +from typing import Dict +from typing import List +from typing import Set +from typing import Union import zope.component import zope.interface -try: - import apacheconfig - HAS_APACHECONFIG = True -except ImportError: # pragma: no cover - HAS_APACHECONFIG = False from acme import challenges -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Union from certbot import errors from certbot import interfaces from certbot import util @@ -41,6 +36,13 @@ from certbot_apache._internal import http_01 from certbot_apache._internal import obj from certbot_apache._internal import parser +try: + import apacheconfig + HAS_APACHECONFIG = True +except ImportError: # pragma: no cover + HAS_APACHECONFIG = False + + logger = logging.getLogger(__name__) @@ -210,23 +212,23 @@ class ApacheConfigurator(common.Installer): super(ApacheConfigurator, self).__init__(*args, **kwargs) # Add name_server association dict - self.assoc = {} # type: Dict[str, obj.VirtualHost] + self.assoc: Dict[str, obj.VirtualHost] = {} # Outstanding challenges - self._chall_out = set() # type: Set[KeyAuthorizationAnnotatedChallenge] + self._chall_out: Set[KeyAuthorizationAnnotatedChallenge] = set() # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {} # Maps enhancements to vhosts we've enabled the enhancement for - self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] + self._enhanced_vhosts: DefaultDict[str, Set[obj.VirtualHost]] = defaultdict(set) # Temporary state for AutoHSTS enhancement - self._autohsts = {} # type: Dict[str, Dict[str, Union[int, float]]] + self._autohsts: Dict[str, Dict[str, Union[int, float]]] = {} # Reverter save notes self.save_notes = "" # Should we use ParserNode implementation instead of the old behavior self.USE_PARSERNODE = use_parsernode # Saves the list of file paths that were parsed initially, and # not added to parser tree by self.conf("vhost-root") for example. - self.parsed_paths = [] # type: List[str] + self.parsed_paths: List[str] = [] # These will be set in the prepare function self._prepared = False self.parser = None @@ -832,7 +834,7 @@ class ApacheConfigurator(common.Installer): :rtype: set """ - all_names = set() # type: Set[str] + all_names: Set[str] = set() vhost_macro = [] @@ -996,8 +998,8 @@ class ApacheConfigurator(common.Installer): """ # Search base config, and all included paths for VirtualHosts - file_paths = {} # type: Dict[str, str] - internal_paths = defaultdict(set) # type: DefaultDict[str, Set[str]] + file_paths: Dict[str, str] = {} + internal_paths: DefaultDict[str, Set[str]] = defaultdict(set) vhs = [] # Make a list of parser paths because the parser_paths # dictionary may be modified during the loop. @@ -2156,7 +2158,7 @@ class ApacheConfigurator(common.Installer): # There can be other RewriteRule directive lines in vhost config. # rewrite_args_dict keys are directive ids and the corresponding value # for each is a list of arguments to that directive. - rewrite_args_dict = defaultdict(list) # type: DefaultDict[str, List[str]] + rewrite_args_dict: DefaultDict[str, List[str]] = defaultdict(list) pat = r'(.*directive\[\d+\]).*' for match in rewrite_path: m = re.match(pat, match) @@ -2250,7 +2252,7 @@ class ApacheConfigurator(common.Installer): if ssl_vhost.aliases: serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases) - rewrite_rule_args = [] # type: List[str] + rewrite_rule_args: List[str] = [] if self.get_version() >= (2, 3, 9): rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END else: diff --git a/certbot-apache/certbot_apache/_internal/dualparser.py b/certbot-apache/certbot_apache/_internal/dualparser.py index eef8f2a0e..c89ff95be 100644 --- a/certbot-apache/certbot_apache/_internal/dualparser.py +++ b/certbot-apache/certbot_apache/_internal/dualparser.py @@ -1,10 +1,10 @@ """ Dual ParserNode implementation """ +from certbot_apache._internal import apacheparser from certbot_apache._internal import assertions from certbot_apache._internal import augeasparser -from certbot_apache._internal import apacheparser -class DualNodeBase(object): +class DualNodeBase: """ Dual parser interface for in development testing. This is used as the base class for dual parser interface classes. This class handles runtime attribute value assertions.""" diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 5ef44fa2e..66d888e97 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -1,9 +1,9 @@ """A class that performs HTTP-01 challenges for Apache""" -import logging import errno +import logging +from typing import List +from typing import Set -from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot.compat import filesystem from certbot.compat import os @@ -57,7 +57,7 @@ class ApacheHttp01(common.ChallengePerformer): self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") - self.moded_vhosts = set() # type: Set[VirtualHost] + self.moded_vhosts: Set[VirtualHost] = set() def perform(self): """Perform all HTTP-01 challenges.""" @@ -93,7 +93,7 @@ class ApacheHttp01(common.ChallengePerformer): self.configurator.enable_mod(mod, temp=True) def _mod_config(self): - selected_vhosts = [] # type: List[VirtualHost] + selected_vhosts: List[VirtualHost] = [] http_port = str(self.configurator.config.http01_port) for chall in self.achalls: # Search for matching VirtualHosts diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 647790c41..38c21b785 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -100,12 +100,9 @@ For this reason the internal representation of data should not ignore the case. """ import abc -import six - -@six.add_metaclass(abc.ABCMeta) -class ParserNode(object): +class ParserNode(object, metaclass=abc.ABCMeta): """ ParserNode is the basic building block of the tree of such nodes, representing the structure of the configuration. It is largely meant to keep @@ -204,9 +201,7 @@ class ParserNode(object): """ -# Linter rule exclusion done because of https://github.com/PyCQA/pylint/issues/179 -@six.add_metaclass(abc.ABCMeta) # pylint: disable=abstract-method -class CommentNode(ParserNode): +class CommentNode(ParserNode, metaclass=abc.ABCMeta): """ CommentNode class is used for representation of comments within the parsed configuration structure. Because of the nature of comments, it is not able @@ -249,8 +244,7 @@ class CommentNode(ParserNode): metadata=kwargs.get('metadata', {})) # pragma: no cover -@six.add_metaclass(abc.ABCMeta) -class DirectiveNode(ParserNode): +class DirectiveNode(ParserNode, metaclass=abc.ABCMeta): """ DirectiveNode class represents a configuration directive within the configuration. It can have zero or more parameters attached to it. Because of the nature of @@ -325,8 +319,7 @@ class DirectiveNode(ParserNode): """ -@six.add_metaclass(abc.ABCMeta) -class BlockNode(DirectiveNode): +class BlockNode(DirectiveNode, metaclass=abc.ABCMeta): """ BlockNode class represents a block of nested configuration directives, comments and other blocks as its children. A BlockNode can have zero or more parameters diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 498766744..21fd042f6 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -1,7 +1,7 @@ """Module contains classes used by the Apache Configurator.""" import re +from typing import Set -from acme.magic_typing import Set from certbot.plugins import common @@ -20,9 +20,6 @@ class Addr(common.Addr): self.is_wildcard() and other.is_wildcard())) return False - def __ne__(self, other): - return not self.__eq__(other) - def __repr__(self): return "certbot_apache._internal.obj.Addr(" + repr(self.tup) + ")" @@ -98,7 +95,7 @@ class Addr(common.Addr): return self.get_addr_obj(port) -class VirtualHost(object): +class VirtualHost: """Represents an Apache Virtualhost. :ivar str filep: file path of VH @@ -140,7 +137,7 @@ class VirtualHost(object): def get_names(self): """Return a set of all names.""" - all_names = set() # type: Set[str] + all_names: Set[str] = set() all_names.update(self.aliases) # Strip out any scheme:// and field from servername if self.name is not None: @@ -191,9 +188,6 @@ class VirtualHost(object): return False - def __ne__(self, other): - return not self.__eq__(other) - def __hash__(self): return hash((self.filep, self.path, tuple(self.addrs), tuple(self.get_names()), @@ -251,7 +245,7 @@ class VirtualHost(object): # already_found acts to keep everything very conservative. # Don't allow multiple ip:ports in same set. - already_found = set() # type: Set[str] + already_found: Set[str] = set() for addr in vhost.addrs: for local_addr in self.addrs: diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 9b2ee54c9..54edfc911 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -1,9 +1,9 @@ """ Distribution specific override class for CentOS family (RHEL, Fedora) """ import logging +from typing import List import zope.interface -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot import util @@ -102,9 +102,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator): loadmods = self.parser.find_dir("LoadModule", "ssl_module", exclude=False) - correct_ifmods = [] # type: List[str] - loadmod_args = [] # type: List[str] - loadmod_paths = [] # type: List[str] + correct_ifmods: List[str] = [] + loadmod_args: List[str] = [] + loadmod_paths: List[str] = [] for m in loadmods: noarg_path = m.rpartition("/")[0] path_args = self.parser.get_all_args(noarg_path) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 4d997545b..1a6af4c4b 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -3,12 +3,9 @@ import copy import fnmatch import logging import re -import sys +from typing import Dict +from typing import List -import six - -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util @@ -17,7 +14,7 @@ from certbot_apache._internal import constants logger = logging.getLogger(__name__) -class ApacheParser(object): +class ApacheParser: """Class handles the fine details of parsing the Apache Configuration. .. todo:: Make parsing general... remove sites-available etc... @@ -51,9 +48,9 @@ class ApacheParser(object): "version 1.2.0 or higher, please make sure you have you have " "those installed.") - self.modules = {} # type: Dict[str, str] - self.parser_paths = {} # type: Dict[str, List[str]] - self.variables = {} # type: Dict[str, str] + self.modules: Dict[str, str] = {} + self.parser_paths: Dict[str, List[str]] = {} + self.variables: Dict[str, str] = {} # Find configuration root and make sure augeas can parse it. self.root = os.path.abspath(root) @@ -266,7 +263,7 @@ class ApacheParser(object): the iteration issue. Else... parse and enable mods at same time. """ - mods = {} # type: Dict[str, str] + mods: Dict[str, str] = {} matches = self.find_dir("LoadModule") iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration @@ -275,7 +272,7 @@ class ApacheParser(object): while len(mods) != prev_size: prev_size = len(mods) - for match_name, match_filename in six.moves.zip( + for match_name, match_filename in zip( iterator, iterator): mod_name = self.get_arg(match_name) mod_filename = self.get_arg(match_filename) @@ -553,7 +550,7 @@ class ApacheParser(object): else: arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg) - ordered_matches = [] # type: List[str] + ordered_matches: List[str] = [] # TODO: Wildcards should be included in alphabetical order # https://httpd.apache.org/docs/2.4/mod/core.html#include @@ -738,9 +735,6 @@ class ApacheParser(object): :rtype: str """ - if sys.version_info < (3, 6): - # This strips off final /Z(?ms) - return fnmatch.translate(clean_fn_match)[:-7] # pragma: no cover # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z return fnmatch.translate(clean_fn_match)[4:-3] # pragma: no cover diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f129343b3..43165a9dd 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index abe72a5d0..83224fc98 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -107,7 +107,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- def test_set_parameters(self): servernames = self.config.parser_root.find_directives("servername") - names = [] # type: List[str] + names: List[str] = [] for servername in servernames: names += servername.parameters self.assertFalse("going_to_set_this" in names) diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index d15600215..db1c46b52 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -7,7 +7,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six # pylint: disable=unused-import # six is used in mock.patch() from certbot import errors from certbot_apache._internal import constants diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 6753995c6..302bf0023 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -10,7 +10,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six # pylint: disable=unused-import # six is used in mock.patch() from acme import challenges from certbot import achallenges @@ -726,7 +725,7 @@ class MultipleVhostsTest(util.ApacheTest): # This calls open self.config.reverter.register_file_creation = mock.Mock() mock_open.side_effect = IOError - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertRaises( errors.PluginError, self.config.make_vhost_ssl, self.vh_truth[0]) @@ -1834,7 +1833,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): def test_open_module_file(self): mock_open = mock.mock_open(read_data="testing 12 3") - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertEqual(self.config._open_module_file("/nonsense/"), "testing 12 3") if __name__ == "__main__": diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 696cd4a54..06e19e445 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -26,7 +26,7 @@ class ApacheHttp01Test(util.ApacheTest): super(ApacheHttp01Test, self).setUp(*args, **kwargs) self.account_key = self.rsa512jwk - self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] + self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = [] vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") # Takes the vhosts for encryption-example.demo, certbot.demo diff --git a/certbot-auto b/certbot-auto index 002fd5ffc..24af0b4c3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.12.0" +LE_AUTO_VERSION="1.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -806,6 +806,7 @@ elif [ -f /etc/mageia-release ]; then NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -1487,18 +1488,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-ci/certbot_integration_tests/assets/hook.py b/certbot-ci/certbot_integration_tests/assets/hook.py index 483892a93..c11704f47 100755 --- a/certbot-ci/certbot_integration_tests/assets/hook.py +++ b/certbot-ci/certbot_integration_tests/assets/hook.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function import os import sys diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index b9854b402..b052e375d 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -7,7 +7,7 @@ import tempfile from certbot_integration_tests.utils import certbot_call -class IntegrationTestsContext(object): +class IntegrationTestsContext: """General fixture describing a certbot integration tests context""" def __init__(self, request): self.request = request diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 2d3d93669..0b649acdc 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -1,5 +1,4 @@ """Module executing integration tests against certbot core.""" -from __future__ import print_function import os from os.path import exists diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index 230fb0eda..6fc77480d 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -6,7 +6,6 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ -from __future__ import print_function import contextlib import subprocess import sys diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py index 8a2d48a50..2c52c8523 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -1,8 +1,8 @@ """Module executing integration tests against certbot with nginx plugin.""" import os import ssl - from typing import List + import pytest from certbot_integration_tests.nginx_tests import context as nginx_context @@ -32,8 +32,8 @@ def test_context(request): '--preferred-challenges', 'http' ], {'default_server': False}), ], indirect=['context']) -def test_certificate_deployment(certname_pattern, params, context): - # type: (str, List[str], nginx_context.IntegrationTestsContext) -> None +def test_certificate_deployment(certname_pattern: str, params: List[str], + context: nginx_context.IntegrationTestsContext) -> None: """ Test various scenarios to deploy a certificate to nginx using certbot. """ diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py index bdedee1fe..c8024b21d 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -1,7 +1,7 @@ """Module to handle the context of RFC2136 integration tests.""" -import tempfile from contextlib import contextmanager +import tempfile from pkg_resources import resource_filename from pytest import skip diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index bbbdd196b..ceebbe7ed 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -1,6 +1,5 @@ #!/usr/bin/env python """Module to setup an ACME CA server environment able to run multiple tests in parallel""" -from __future__ import print_function import argparse import errno @@ -12,18 +11,18 @@ import subprocess import sys import tempfile import time - from typing import List + import requests +# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils import misc from certbot_integration_tests.utils import pebble_artifacts from certbot_integration_tests.utils import proxy -# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils.constants import * -class ACMEServer(object): +class ACMEServer: """ ACMEServer configures and handles the lifecycle of an ACME CA server and an HTTP reverse proxy instance, to allow parallel execution of integration tests against the unique http-01 port @@ -52,7 +51,7 @@ class ACMEServer(object): self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' self._proxy = http_proxy self._workspace = tempfile.mkdtemp() - self._processes = [] # type: List[subprocess.Popen] + self._processes: List[subprocess.Popen] = [] self._stdout = sys.stdout if stdout else open(os.devnull, 'w') self._dns_server = dns_server self._http_01_port = http_01_port diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py index c9e46cdc7..b319eca4c 100755 --- a/certbot-ci/certbot_integration_tests/utils/certbot_call.py +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -1,6 +1,5 @@ #!/usr/bin/env python """Module to call certbot in test mode""" -from __future__ import absolute_import import os import subprocess diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index 416f6567e..31e7aee18 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -1,7 +1,5 @@ #!/usr/bin/env python """Module to setup an RFC2136-capable DNS server""" -from __future__ import print_function - import os import os.path import shutil @@ -21,7 +19,7 @@ BIND_BIND_ADDRESS = ("127.0.0.1", 45953) BIND_TEST_QUERY = bytearray.fromhex("0011cb37000000010000000000000000010003") -class DNSServer(object): +class DNSServer: """ DNSServer configures and handles the lifetime of an RFC2136-capable server. DNServer provides access to the dns_xdist parameter, listing the address and port @@ -40,7 +38,7 @@ class DNSServer(object): self.bind_root = tempfile.mkdtemp() - self.process = None # type: subprocess.Popen + self.process: subprocess.Popen = None self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]} @@ -113,8 +111,7 @@ class DNSServer(object): self.stop() raise - def _wait_until_ready(self, attempts=30): - # type: (int) -> None + def _wait_until_ready(self, attempts: int = 30) -> None: """ Polls the DNS server over TCP until it gets a response, or until it runs out of attempts and raises a ValueError. diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index 799b079fe..2fac494f2 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -4,10 +4,12 @@ or outside during setup/teardown of the integration tests environment. """ import contextlib import errno +import http.server as SimpleHTTPServer import multiprocessing import os import re import shutil +import socketserver import stat import sys import tempfile @@ -23,11 +25,9 @@ from cryptography.x509 import load_pem_x509_certificate from OpenSSL import crypto import pkg_resources import requests -from six.moves import SimpleHTTPServer -from six.moves import socketserver -from certbot_integration_tests.utils.constants import \ - PEBBLE_ALTERNATE_ROOTS, PEBBLE_MANAGEMENT_URL +from certbot_integration_tests.utils.constants import PEBBLE_ALTERNATE_ROOTS +from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL RSA_KEY_TYPE = 'rsa' ECDSA_KEY_TYPE = 'ecdsa' @@ -311,7 +311,7 @@ def echo(keyword, path=None): if not re.match(r'^\w+$', keyword): raise ValueError('Error, keyword `{0}` is not a single keyword.' .format(keyword)) - return '{0} -c "from __future__ import print_function; print(\'{1}\')"{2}'.format( + return '{0} -c "print(\'{1}\')"{2}'.format( os.path.basename(sys.executable), keyword, ' >> "{0}"'.format(path) if path else '') diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index cd62e1a7f..918a5fd04 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -7,7 +7,8 @@ import stat import pkg_resources import requests -from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT, MOCK_OCSP_SERVER_PORT +from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT +from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT PEBBLE_VERSION = 'v2.3.0' ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py index b86e1cbc9..4db72998f 100755 --- a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py @@ -4,6 +4,7 @@ This runnable module interfaces itself with the Pebble management interface in o to serve a mock OCSP responder during integration tests against Pebble. """ import datetime +import http.server as BaseHTTPServer import re from cryptography import x509 @@ -13,7 +14,6 @@ from cryptography.hazmat.primitives import serialization from cryptography.x509 import ocsp from dateutil import parser import requests -from six.moves import BaseHTTPServer from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL diff --git a/certbot-ci/certbot_integration_tests/utils/proxy.py b/certbot-ci/certbot_integration_tests/utils/proxy.py index 225f98e6e..7c46e640d 100644 --- a/certbot-ci/certbot_integration_tests/utils/proxy.py +++ b/certbot-ci/certbot_integration_tests/utils/proxy.py @@ -1,12 +1,12 @@ #!/usr/bin/env python # pylint: disable=missing-module-docstring +import http.server as BaseHTTPServer import json import re import sys import requests -from six.moves import BaseHTTPServer from certbot_integration_tests.utils.misc import GracefulTCPServer diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index 3277df1c0..9f9c1f462 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -18,7 +18,6 @@ install_requires = [ 'python-dateutil', 'pyyaml', 'requests', - 'six' ] # Add pywin32 on Windows platforms to handle low-level system calls. diff --git a/certbot-ci/snap_integration_tests/dns_tests/test_main.py b/certbot-ci/snap_integration_tests/dns_tests/test_main.py index 016355334..721352c04 100644 --- a/certbot-ci/snap_integration_tests/dns_tests/test_main.py +++ b/certbot-ci/snap_integration_tests/dns_tests/test_main.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -import pytest -import subprocess import glob import os import re +import subprocess + +import pytest @pytest.fixture(autouse=True, scope="module") diff --git a/certbot-ci/windows_installer_integration_tests/conftest.py b/certbot-ci/windows_installer_integration_tests/conftest.py index c6a89c323..8a9de057f 100644 --- a/certbot-ci/windows_installer_integration_tests/conftest.py +++ b/certbot-ci/windows_installer_integration_tests/conftest.py @@ -6,7 +6,7 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ -from __future__ import print_function + import os ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) diff --git a/certbot-ci/windows_installer_integration_tests/test_main.py b/certbot-ci/windows_installer_integration_tests/test_main.py index c8c347aa8..f699b736a 100644 --- a/certbot-ci/windows_installer_integration_tests/test_main.py +++ b/certbot-ci/windows_installer_integration_tests/test_main.py @@ -1,8 +1,8 @@ import os +import re +import subprocess import time import unittest -import subprocess -import re @unittest.skipIf(os.name != 'nt', reason='Windows installer tests must be run on Windows.') diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index 34aa9133f..bf768f8f8 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -11,7 +11,7 @@ from certbot_compatibility_test import util logger = logging.getLogger(__name__) -class Proxy(object): +class Proxy: """A common base for compatibility test configurators""" @classmethod diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py index 7cb4e9722..d7b7a120e 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py @@ -2,10 +2,10 @@ import os import shutil import subprocess +from typing import Set import zope.interface -from acme.magic_typing import Set from certbot._internal import configuration from certbot_compatibility_test import errors from certbot_compatibility_test import interfaces @@ -68,7 +68,7 @@ def _get_server_root(config): def _get_names(config): """Returns all and testable domain names in config""" - all_names = set() # type: Set[str] + all_names: Set[str] = set() for root, _dirs, files in os.walk(config): for this_file in files: update_names = _get_server_names(root, this_file) diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index f11b9fdf8..4bb16e63f 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -8,6 +8,8 @@ import shutil import sys import tempfile import time +from typing import List +from typing import Tuple import OpenSSL from urllib3.util import connection @@ -15,8 +17,6 @@ from urllib3.util import connection from acme import challenges from acme import crypto_util from acme import messages -from acme.magic_typing import List -from acme.magic_typing import Tuple from certbot import achallenges from certbot import errors as le_errors from certbot.tests import acme_util @@ -178,7 +178,7 @@ def test_enhancements(plugin, domains): "enhancements") return False - domains_and_info = [(domain, []) for domain in domains] # type: List[Tuple[str, List[bool]]] + domains_and_info: List[Tuple[str, List[bool]]] = [(domain, []) for domain in domains] for domain, info in domains_and_info: try: diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index bfa03f22d..226b8585b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -3,8 +3,6 @@ import logging import socket import requests -import six -from six.moves import xrange from acme import crypto_util from acme import errors as acme_errors @@ -12,18 +10,18 @@ from acme import errors as acme_errors logger = logging.getLogger(__name__) -class Validator(object): +class Validator: """Collection of functions to test a live webserver's configuration""" def certificate(self, cert, name, alt_host=None, port=443): """Verifies the certificate presented at name is cert""" if alt_host is None: host = socket.gethostbyname(name).encode() - elif isinstance(alt_host, six.binary_type): + elif isinstance(alt_host, bytes): host = alt_host else: host = alt_host.encode() - name = name if isinstance(name, six.binary_type) else name.encode() + name = name if isinstance(name, bytes) else name.encode() try: presented_cert = crypto_util.probe_sni(name, host, port) @@ -62,7 +60,7 @@ class Validator(object): else: response = requests.get(url, allow_redirects=False) - return response.status_code in xrange(300, 309) + return response.status_code in range(300, 309) def hsts(self, name): """Test for HTTP Strict Transport Security header""" diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 0236773f0..3e9e6a7c2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,12 +3,11 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' install_requires = [ 'certbot', 'certbot-apache', - 'six', 'requests', 'zope.interface', ] diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index 5978af85c..034d7bbb2 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -1,13 +1,12 @@ """DNS Authenticator for Cloudflare.""" import logging +from typing import Any +from typing import Dict +from typing import List import CloudFlare import zope.interface -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import List - from certbot import errors from certbot import interfaces from certbot.plugins import dns_common @@ -85,7 +84,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _CloudflareClient(self.credentials.conf('email'), self.credentials.conf('api-key')) -class _CloudflareClient(object): +class _CloudflareClient: """ Encapsulates all communication with the Cloudflare API. """ @@ -173,7 +172,7 @@ class _CloudflareClient(object): """ zone_name_guesses = dns_common.base_domain_name_guesses(domain) - zones = [] # type: List[Dict[str, Any]] + zones: List[Dict[str, Any]] = [] code = msg = None for zone_name in zone_name_guesses: diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index eab6cdb70..2e5215f9c 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 83513ef7c..44bada6bc 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index e0c9561a2..cb5012fb7 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -21,6 +21,7 @@ class Authenticator(dns_common.DNSAuthenticator): description = 'Obtain certificates using a DNS TXT record (if you are ' + \ 'using DigitalOcean for DNS).' + ttl = 30 def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) @@ -45,7 +46,8 @@ class Authenticator(dns_common.DNSAuthenticator): ) def _perform(self, domain, validation_name, validation): - self._get_digitalocean_client().add_txt_record(domain, validation_name, validation) + self._get_digitalocean_client().add_txt_record(domain, validation_name, validation, + self.ttl) def _cleanup(self, domain, validation_name, validation): self._get_digitalocean_client().del_txt_record(domain, validation_name, validation) @@ -54,7 +56,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _DigitalOceanClient(self.credentials.conf('token')) -class _DigitalOceanClient(object): +class _DigitalOceanClient: """ Encapsulates all communication with the DigitalOcean API. """ @@ -62,13 +64,15 @@ class _DigitalOceanClient(object): def __init__(self, token): self.manager = digitalocean.Manager(token=token) - def add_txt_record(self, domain_name, record_name, record_content): + def add_txt_record(self, domain_name: str, record_name: str, record_content: str, + record_ttl: int): """ Add a TXT record using the supplied information. :param str domain_name: The domain to use to associate the record with. :param str record_name: The record name (typically beginning with '_acme-challenge.'). :param str record_content: The record content (typically the challenge validation). + :param int record_ttl: The record TTL. :raises certbot.errors.PluginError: if an error occurs communicating with the DigitalOcean API """ @@ -89,7 +93,8 @@ class _DigitalOceanClient(object): result = domain.create_new_domain_record( type='TXT', name=self._compute_record_name(domain, record_name), - data=record_content) + data=record_content, + ttl=record_ttl) # ttl kwarg is only effective starting python-digitalocean 1.15.0 record_id = result['domain_record']['id'] @@ -99,7 +104,7 @@ class _DigitalOceanClient(object): raise errors.PluginError('Error adding TXT record using the DigitalOcean API: {0}' .format(e)) - def del_txt_record(self, domain_name, record_name, record_content): + def del_txt_record(self, domain_name: str, record_name: str, record_content: str): """ Delete a TXT record using the supplied information. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 8c6ac78d5..106eb3cac 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,14 +4,13 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'python-digitalocean>=1.11', + 'python-digitalocean>=1.11', # 1.15.0 or newer is recommended for TTL support 'setuptools>=39.0.1', - 'six>=1.11.0', 'zope.interface', ] diff --git a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py index a752f52d0..84bb35b1f 100644 --- a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py @@ -40,7 +40,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def test_perform(self): self.auth.perform([self.achall]) - expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] + expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY, 30)] self.assertEqual(expected, self.mock_client.mock_calls) def test_cleanup(self): @@ -58,6 +58,7 @@ class DigitalOceanClientTest(unittest.TestCase): record_prefix = "_acme-challenge" record_name = record_prefix + "." + DOMAIN record_content = "bar" + record_ttl = 60 def setUp(self): from certbot_dns_digitalocean._internal.dns_digitalocean import _DigitalOceanClient @@ -78,25 +79,27 @@ class DigitalOceanClientTest(unittest.TestCase): self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock] - self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content) + self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content, + self.record_ttl) domain_mock.create_new_domain_record.assert_called_with(type='TXT', name=self.record_prefix, - data=self.record_content) + data=self.record_content, + ttl=self.record_ttl) def test_add_txt_record_fail_to_find_domain(self): self.manager.get_all_domains.return_value = [] self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_add_txt_record_error_finding_domain(self): self.manager.get_all_domains.side_effect = API_ERROR self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_add_txt_record_error_creating_record(self): domain_mock = mock.MagicMock() @@ -107,7 +110,7 @@ class DigitalOceanClientTest(unittest.TestCase): self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_del_txt_record(self): first_record_mock = mock.MagicMock() diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index f1fcfd11d..99a3c92a3 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 185048a2d..b126b4e8f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 0ae9c1bf7..cada7a85e 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index 4b0d91463..363c5e079 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -76,7 +76,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _GoogleClient(self.conf('credentials')) -class _GoogleClient(object): +class _GoogleClient: """ Encapsulates all communication with the Google Cloud DNS API. """ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index b16d014c6..e8b26eceb 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 396a6c8bd..7de5f1d67 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -401,7 +401,7 @@ class GoogleClientTest(unittest.TestCase): self.assertRaises(ServerNotFoundError, _GoogleClient.get_project_id) -class DummyResponse(object): +class DummyResponse: """ Dummy object to create a fake HTTPResponse (the actual one requires a socket and we only need the status attribute) diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 21ccf9d42..63da31df2 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 2312d6fcc..61b9dfa78 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 658027b9a..d60412f27 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index b4f73ddb4..4d6131ff7 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py index 57e9506f2..adebdd963 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py @@ -88,7 +88,7 @@ class Authenticator(dns_common.DNSAuthenticator): dns.tsig.HMAC_MD5)) -class _RFC2136Client(object): +class _RFC2136Client: """ Encapsulates all communication with the target DNS server. """ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index ce74611cd..18d006732 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py index 6250d2274..4dda13f1d 100644 --- a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py @@ -2,15 +2,15 @@ import collections import logging import time +from typing import DefaultDict +from typing import Dict +from typing import List import boto3 from botocore.exceptions import ClientError from botocore.exceptions import NoCredentialsError import zope.interface -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot.plugins import dns_common @@ -39,7 +39,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) self.r53 = boto3.client("route53") - self._resource_records = collections.defaultdict(list) # type: DefaultDict[str, List[Dict[str, str]]] + self._resource_records: DefaultDict[str, List[Dict[str, str]]] = collections.defaultdict(list) def more_info(self): # pylint: disable=missing-function-docstring return "Solve a DNS01 challenge using AWS Route53" diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8def9a702..fdc931bbe 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 6f4f8e506..55f140d76 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 15fbe61f7..aab489315 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-lines """Nginx Configuration""" from distutils.version import LooseVersion import logging @@ -6,6 +7,12 @@ import socket import subprocess import tempfile import time +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Text +from typing import Tuple import OpenSSL import pkg_resources @@ -13,10 +20,6 @@ import zope.interface from acme import challenges from acme import crypto_util as acme_crypto_util -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Text from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -105,18 +108,18 @@ class NginxConfigurator(common.Installer): self.save_notes = "" # For creating new vhosts if no names match - self.new_vhost = None + self.new_vhost: Optional[obj.VirtualHost] = None # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] - self._wildcard_redirect_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {} + self._wildcard_redirect_vhosts: Dict[str, List[obj.VirtualHost]] = {} # Add number of outstanding challenges self._chall_out = 0 # These will be set in the prepare function - self.parser = None + self.parser: Optional[parser.NginxParser] = None self.version = version self.openssl_version = openssl_version self._enhance_func = {"redirect": self._enable_redirect, @@ -377,10 +380,13 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain, allow_port_mismatch, port): + def _vhost_from_duplicated_default(self, domain: str, allow_port_mismatch: bool, port: str + ) -> obj.VirtualHost: """if allow_port_mismatch is False, only server blocks with matching ports will be used as a default server block template. """ + assert self.parser is not None # prepare should already have been called here + if self.new_vhost is None: default_vhost = self._get_default_vhost(domain, allow_port_mismatch, port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, @@ -509,7 +515,7 @@ class NginxConfigurator(common.Installer): match['rank'] += NO_SSL_MODIFIER return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): + def choose_redirect_vhosts(self, target_name: str, port: str) -> List[obj.VirtualHost]: """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -523,9 +529,6 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param str port: port number - :param bool create_if_no_match: If we should create a new vhost from default - when there is no match found. If we can't choose a default, raise a - MisconfigurationError. :returns: vhosts associated with name :rtype: list of :class:`~certbot_nginx._internal.obj.VirtualHost` @@ -538,32 +541,75 @@ class NginxConfigurator(common.Installer): else: matches = self._get_redirect_ranked_matches(target_name, port) vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] - if not vhosts and create_if_no_match: - vhosts = [self._vhost_from_duplicated_default(target_name, False, port)] return vhosts - def _port_matches(self, test_port, matching_port): + def choose_auth_vhosts(self, target_name: str) -> Tuple[List[obj.VirtualHost], + List[obj.VirtualHost]]: + """Returns a list of HTTP and HTTPS vhosts with a server_name matching target_name. + + If no HTTP vhost exists, one will be cloned from the default vhost. If that fails, no HTTP + vhost will be returned. + + :param str target_name: non-wildcard domain name + + :returns: tuple of HTTP and HTTPS virtualhosts + :rtype: tuple of :class:`~certbot_nginx._internal.obj.VirtualHost` + + """ + vhosts = [m['vhost'] for m in self._get_ranked_matches(target_name) if m and 'vhost' in m] + http_vhosts = [vh for vh in vhosts if + self._vhost_listening(vh, str(self.config.http01_port), False)] + https_vhosts = [vh for vh in vhosts if + self._vhost_listening(vh, str(self.config.https_port), True)] + + # If no HTTP vhost matches, try create one from the default_server on http01_port. + if not http_vhosts: + try: + http_vhosts = [self._vhost_from_duplicated_default(target_name, False, + str(self.config.http01_port))] + except errors.MisconfigurationError: + http_vhosts = [] + + return http_vhosts, https_vhosts + + def _port_matches(self, test_port: str, matching_port: str) -> bool: # test_port is a number, matching is a number or "" or None if matching_port == "" or matching_port is None: # if no port is specified, Nginx defaults to listening on port 80. return test_port == self.DEFAULT_LISTEN_PORT return test_port == matching_port - def _vhost_listening_on_port_no_ssl(self, vhost, port): - found_matching_port = False - if not vhost.addrs: - # if there are no listen directives at all, Nginx defaults to - # listening on port 80. - found_matching_port = (port == self.DEFAULT_LISTEN_PORT) - else: - for addr in vhost.addrs: - if self._port_matches(port, addr.get_port()) and not addr.ssl: - found_matching_port = True + def _vhost_listening(self, vhost: obj.VirtualHost, port: str, ssl: bool) -> bool: + """Tests whether a vhost has an address listening on a port with SSL enabled or disabled. - if found_matching_port: - # make sure we don't have an 'ssl on' directive - return not self.parser.has_ssl_on_directive(vhost) - return False + :param `obj.VirtualHost` vhost: The vhost whose addresses will be tested + :param port str: The port number as a string that the address should be bound to + :param bool ssl: Whether SSL should be enabled or disabled on the address + + :returns: Whether the vhost has an address listening on the port and protocol. + :rtype: bool + + """ + assert self.parser is not None # prepare should already have been called here + + # if the 'ssl on' directive is present on the vhost, all its addresses have SSL enabled + all_addrs_are_ssl = self.parser.has_ssl_on_directive(vhost) + + # if we want ssl vhosts: either 'ssl on' or 'addr.ssl' should be enabled + # if we want plaintext vhosts: neither 'ssl on' nor 'addr.ssl' should be enabled + _ssl_matches = lambda addr: addr.ssl or all_addrs_are_ssl if ssl else \ + not addr.ssl and not all_addrs_are_ssl + + # if there are no listen directives at all, Nginx defaults to + # listening on port 80. + if not vhost.addrs: + return port == self.DEFAULT_LISTEN_PORT and ssl == all_addrs_are_ssl + + return any(self._port_matches(port, addr.get_port()) and _ssl_matches(addr) + for addr in vhost.addrs) + + def _vhost_listening_on_port_no_ssl(self, vhost: obj.VirtualHost, port: str) -> bool: + return self._vhost_listening(vhost, port, False) def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -595,7 +641,7 @@ class NginxConfigurator(common.Installer): :rtype: set """ - all_names = set() # type: Set[str] + all_names: Set[str] = set() for vhost in self.parser.get_vhosts(): try: @@ -1176,7 +1222,7 @@ def nginx_restart(nginx_ctl, nginx_conf, sleep_duration): """ try: - reload_output = u"" # type: Text + reload_output: Text = u"" with tempfile.TemporaryFile() as out: proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"], env=util.env_no_snap_for_external_calls(), diff --git a/certbot-nginx/certbot_nginx/_internal/constants.py b/certbot-nginx/certbot_nginx/_internal/constants.py index 1f058e7ef..52f4d8238 100644 --- a/certbot-nginx/certbot_nginx/_internal/constants.py +++ b/certbot-nginx/certbot_nginx/_internal/constants.py @@ -1,8 +1,7 @@ """nginx plugin constants.""" import platform - -from acme.magic_typing import Any -from acme.magic_typing import Dict +from typing import Any +from typing import Dict FREEBSD_DARWIN_SERVER_ROOT = "/usr/local/etc/nginx" LINUX_SERVER_ROOT = "/etc/nginx" @@ -15,11 +14,11 @@ elif platform.system() in ('NetBSD',): else: server_root_tmp = LINUX_SERVER_ROOT -CLI_DEFAULTS = dict( +CLI_DEFAULTS: Dict[str, Any] = dict( server_root=server_root_tmp, ctl="nginx", sleep_seconds=1 -) # type: Dict[str, Any] +) """CLI defaults.""" diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index 40f994988..abcdfbd53 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -2,9 +2,11 @@ import io import logging +from typing import List +from typing import Optional from acme import challenges -from acme.magic_typing import List +from certbot import achallenges from certbot import errors from certbot.compat import os from certbot.plugins import common @@ -111,7 +113,7 @@ class NginxHttp01(common.ChallengePerformer): :returns: list of :class:`certbot_nginx._internal.obj.Addr` to apply :rtype: list """ - addresses = [] # type: List[obj.Addr] + addresses: List[obj.Addr] = [] default_addr = "%s" % self.configurator.config.http01_port ipv6_addr = "[::]:{0}".format( self.configurator.config.http01_port) @@ -138,13 +140,12 @@ class NginxHttp01(common.ChallengePerformer): def _get_validation_path(self, achall): return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token")) - def _make_server_block(self, achall): + def _make_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge) -> List: """Creates a server block for a challenge. + :param achall: Annotated HTTP-01 challenge - :type achall: - :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` - :param list addrs: addresses of challenged domain - :class:`list` of type :class:`~nginx.obj.Addr` + :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :returns: server block for the challenge host :rtype: list """ @@ -172,34 +173,35 @@ class NginxHttp01(common.ChallengePerformer): return location_directive - def _make_or_mod_server_block(self, achall): - """Modifies a server block to respond to a challenge. + def _make_or_mod_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge + ) -> Optional[List]: + """Modifies server blocks to respond to a challenge. Returns a new HTTP server block + to add to the configuration if an existing one can't be found. :param achall: Annotated HTTP-01 challenge - :type achall: - :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + + :returns: new server block to be added, if any + :rtype: list """ - try: - vhosts = self.configurator.choose_redirect_vhosts(achall.domain, - '%i' % self.configurator.config.http01_port, create_if_no_match=True) - except errors.MisconfigurationError: + http_vhosts, https_vhosts = self.configurator.choose_auth_vhosts(achall.domain) + + new_vhost: Optional[list] = None + if not http_vhosts: # Couldn't find either a matching name+port server block # or a port+default_server block, so create a dummy block - return self._make_server_block(achall) + new_vhost = self._make_server_block(achall) - # len is max 1 because Nginx doesn't authenticate wildcards - # if len were or vhosts None, we would have errored - vhost = vhosts[0] + # Modify any existing server blocks + for vhost in set(http_vhosts + https_vhosts): + location_directive = [self._location_directive_for_achall(achall)] - # Modify existing server block - location_directive = [self._location_directive_for_achall(achall)] + self.configurator.parser.add_server_directives(vhost, location_directive) - self.configurator.parser.add_server_directives(vhost, - location_directive) + rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', + ' ', '$1', ' ', 'break']] + self.configurator.parser.add_server_directives(vhost, + rewrite_directive, insert_at_top=True) - rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', - ' ', '$1', ' ', 'break']] - self.configurator.parser.add_server_directives(vhost, - rewrite_directive, insert_at_top=True) - return None + return new_vhost diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index 5f723dcef..787f7c363 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -2,6 +2,8 @@ # Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed) import copy import logging +from typing import Any +from typing import IO from pyparsing import Combine from pyparsing import Forward @@ -15,12 +17,11 @@ from pyparsing import restOfLine from pyparsing import stringEnd from pyparsing import White from pyparsing import ZeroOrMore -import six -from acme.magic_typing import IO, Any # pylint: disable=unused-import logger = logging.getLogger(__name__) -class RawNginxParser(object): + +class RawNginxParser: # pylint: disable=pointless-statement """A class that parses nginx configuration with pyparsing.""" @@ -70,7 +71,7 @@ class RawNginxParser(object): """Returns the parsed tree as a list.""" return self.parse().asList() -class RawNginxDumper(object): +class RawNginxDumper: """A class that dumps nginx configuration from the provided tree.""" def __init__(self, blocks): self.blocks = blocks @@ -79,7 +80,7 @@ class RawNginxDumper(object): """Iterates the dumped nginx content.""" blocks = blocks or self.blocks for b0 in blocks: - if isinstance(b0, six.string_types): + if isinstance(b0, str): yield b0 continue item = copy.deepcopy(b0) @@ -96,7 +97,7 @@ class RawNginxDumper(object): yield '}' else: # not a block - list of strings semicolon = ";" - if isinstance(item[0], six.string_types) and item[0].strip() == '#': # comment + if isinstance(item[0], str) and item[0].strip() == '#': # comment semicolon = "" yield "".join(item) + semicolon @@ -105,56 +106,8 @@ class RawNginxDumper(object): return ''.join(self) -# Shortcut functions to respect Python's serialization interface -# (like pyyaml, picker or json) +spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == '' -def loads(source): - """Parses from a string. - - :param str source: The string to parse - :returns: The parsed tree - :rtype: list - - """ - return UnspacedList(RawNginxParser(source).as_list()) - - -def load(_file): - """Parses from a file. - - :param file _file: The file to parse - :returns: The parsed tree - :rtype: list - - """ - return loads(_file.read()) - - -def dumps(blocks): - # type: (UnspacedList) -> six.text_type - """Dump to a Unicode string. - - :param UnspacedList block: The parsed tree - :rtype: six.text_type - - """ - return six.text_type(RawNginxDumper(blocks.spaced)) - - -def dump(blocks, _file): - # type: (UnspacedList, IO[Any]) -> None - """Dump to a file. - - :param UnspacedList block: The parsed tree - :param IO[Any] _file: The file stream to dump to. It must be opened with - Unicode encoding. - :rtype: None - - """ - _file.write(dumps(blocks)) - - -spacey = lambda x: (isinstance(x, six.string_types) and x.isspace()) or x == '' class UnspacedList(list): """Wrap a list [of lists], making any whitespace entries magically invisible""" @@ -274,3 +227,50 @@ class UnspacedList(list): idx -= 1 pos += 1 return idx0 + spaces + + +# Shortcut functions to respect Python's serialization interface +# (like pyyaml, picker or json) + +def loads(source): + """Parses from a string. + + :param str source: The string to parse + :returns: The parsed tree + :rtype: list + + """ + return UnspacedList(RawNginxParser(source).as_list()) + + +def load(_file): + """Parses from a file. + + :param file _file: The file to parse + :returns: The parsed tree + :rtype: list + + """ + return loads(_file.read()) + + +def dumps(blocks: UnspacedList) -> str: + """Dump to a Unicode string. + + :param UnspacedList block: The parsed tree + :rtype: six.text_type + + """ + return str(RawNginxDumper(blocks.spaced)) + + +def dump(blocks: UnspacedList, _file: IO[Any]) -> None: + """Dump to a file. + + :param UnspacedList block: The parsed tree + :param IO[Any] _file: The file stream to dump to. It must be opened with + Unicode encoding. + :rtype: None + + """ + _file.write(dumps(blocks)) diff --git a/certbot-nginx/certbot_nginx/_internal/obj.py b/certbot-nginx/certbot_nginx/_internal/obj.py index 4e0d8cf35..1511cba6d 100644 --- a/certbot-nginx/certbot_nginx/_internal/obj.py +++ b/certbot-nginx/certbot_nginx/_internal/obj.py @@ -1,7 +1,6 @@ """Module contains classes used by the Nginx Configurator.""" import re -import six from certbot.plugins import common @@ -144,7 +143,7 @@ class Addr(common.Addr): return False -class VirtualHost(object): +class VirtualHost: """Represents an Nginx Virtualhost. :ivar str filep: file path of VH @@ -211,7 +210,7 @@ class VirtualHost(object): def contains_list(self, test): """Determine if raw server block contains test list at top level """ - for i in six.moves.range(0, len(self.raw) - len(test) + 1): + for i in range(0, len(self.raw) - len(test) + 1): if self.raw[i:i + len(test)] == test: return True return False @@ -250,7 +249,7 @@ def _find_directive(directives, directive_name, match_content=None): """Find a directive of type directive_name in directives. If match_content is given, Searches for `match_content` in the directive arguments. """ - if not directives or isinstance(directives, six.string_types): + if not directives or isinstance(directives, str): return None # If match_content is None, just match on directive type. Otherwise, match on diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 72091b03f..8b353c095 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -5,15 +5,15 @@ import glob import io import logging import re +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Tuple +from typing import Union import pyparsing -import six -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Tuple -from acme.magic_typing import Union from certbot import errors from certbot.compat import os from certbot_nginx._internal import nginxparser @@ -22,7 +22,7 @@ from certbot_nginx._internal import obj logger = logging.getLogger(__name__) -class NginxParser(object): +class NginxParser: """Class handles the fine details of parsing the Nginx Configuration. :ivar str root: Normalized absolute path to the server root @@ -32,7 +32,7 @@ class NginxParser(object): """ def __init__(self, root): - self.parsed = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] + self.parsed: Dict[str, Union[List, nginxparser.UnspacedList]] = {} self.root = os.path.abspath(root) self.config_root = self._find_config_root() @@ -94,7 +94,7 @@ class NginxParser(object): """ servers = self._get_raw_servers() - addr_to_ssl = {} # type: Dict[Tuple[str, str], bool] + addr_to_ssl: Dict[Tuple[str, str], bool] = {} for filename in servers: for server, _ in servers[filename]: # Parse the server block to save addr info @@ -106,12 +106,11 @@ class NginxParser(object): addr_to_ssl[addr_tuple] = addr.ssl or addr_to_ssl[addr_tuple] return addr_to_ssl - def _get_raw_servers(self): + def _get_raw_servers(self) -> Dict: # pylint: disable=cell-var-from-loop - # type: () -> Dict """Get a map of unparsed all server blocks """ - servers = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] + servers: Dict[str, Union[List, nginxparser.UnspacedList]] = {} for filename in self.parsed: tree = self.parsed[filename] servers[filename] = [] @@ -362,8 +361,9 @@ class NginxParser(object): except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) - def duplicate_vhost(self, vhost_template, remove_singleton_listen_params=False, - only_directives=None): + def duplicate_vhost(self, vhost_template: obj.VirtualHost, + remove_singleton_listen_params: bool = False, + only_directives: Optional[List] = None) -> obj.VirtualHost: """Duplicate the vhost in the configuration files. :param :class:`~certbot_nginx._internal.obj.VirtualHost` vhost_template: The vhost @@ -549,7 +549,7 @@ def _is_include_directive(entry): """ return (isinstance(entry, list) and len(entry) == 2 and entry[0] == 'include' and - isinstance(entry[1], six.string_types)) + isinstance(entry[1], str)) def _is_ssl_on_directive(entry): """Checks if an nginx parsed entry is an 'ssl on' directive. @@ -654,7 +654,7 @@ def _add_directive(block, directive, insert_at_top): directive_name = directive[0] def can_append(loc, dir_name): """ Can we append this directive to the block? """ - return loc is None or (isinstance(dir_name, six.string_types) + return loc is None or (isinstance(dir_name, str) and dir_name in REPEATABLE_DIRECTIVES) err_fmt = 'tried to insert directive "{0}" but found conflicting "{1}".' @@ -740,9 +740,9 @@ def _parse_server_raw(server): :rtype: dict """ - addrs = set() # type: Set[obj.Addr] - ssl = False # type: bool - names = set() # type: Set[str] + addrs: Set[obj.Addr] = set() + ssl: bool = False + names: Set[str] = set() apply_ssl_to_all_addrs = False diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 390e18e4d..37392ea80 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -3,10 +3,8 @@ raw lists of tokens from pyparsing. """ import abc import logging +from typing import List -import six - -from acme.magic_typing import List from certbot import errors logger = logging.getLogger(__name__) @@ -14,7 +12,7 @@ COMMENT = " managed by Certbot" COMMENT_BLOCK = ["#", COMMENT] -class Parsable(object): +class Parsable: """ Abstract base class for "Parsable" objects whose underlying representation is a tree of lists. @@ -24,7 +22,7 @@ class Parsable(object): __metaclass__ = abc.ABCMeta def __init__(self, parent=None): - self._data = [] # type: List[object] + self._data: List[object] = [] self._tabs = None self.parent = parent @@ -152,7 +150,7 @@ class Statements(Parsable): if not isinstance(raw_list, list): raise errors.MisconfigurationError("Statements parsing expects a list!") # If there's a trailing whitespace in the list of statements, keep track of it. - if raw_list and isinstance(raw_list[-1], six.string_types) and raw_list[-1].isspace(): + if raw_list and isinstance(raw_list[-1], str) and raw_list[-1].isspace(): self._trailing_whitespace = raw_list[-1] raw_list = raw_list[:-1] self._data = [parse_raw(elem, self, add_spaces) for elem in raw_list] @@ -183,8 +181,8 @@ class Statements(Parsable): def _space_list(list_): """ Inserts whitespace between adjacent non-whitespace tokens. """ - spaced_statement = [] # type: List[str] - for i in reversed(six.moves.xrange(len(list_))): + spaced_statement: List[str] = [] + for i in reversed(range(len(list_))): spaced_statement.insert(0, list_[i]) if i > 0 and not list_[i].isspace() and not list_[i-1].isspace(): spaced_statement.insert(0, " ") @@ -206,7 +204,7 @@ class Sentence(Parsable): :returns: whether this lists is parseable by `Sentence`. """ return isinstance(lists, list) and len(lists) > 0 and \ - all(isinstance(elem, six.string_types) for elem in lists) + all(isinstance(elem, str) for elem in lists) def parse(self, raw_list, add_spaces=False): """ Parses a list of string types into this object. @@ -214,7 +212,7 @@ class Sentence(Parsable): if add_spaces: raw_list = _space_list(raw_list) if not isinstance(raw_list, list) or \ - any(not isinstance(elem, six.string_types) for elem in raw_list): + any(not isinstance(elem, str) for elem in raw_list): raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") self._data = raw_list @@ -272,8 +270,8 @@ class Block(Parsable): """ def __init__(self, parent=None): super(Block, self).__init__(parent) - self.names = None # type: Sentence - self.contents = None # type: Block + self.names: Sentence = None + self.contents: Block = None @staticmethod def should_parse(lists): diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 385f4cc17..8b638f28e 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 5e788e394..9ccc3fc9e 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -39,7 +39,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(12, len(self.config.parser.parsed)) + self.assertEqual(13, len(self.config.parser.parsed)) @mock.patch("certbot_nginx._internal.configurator.util.exe_exists") @mock.patch("certbot_nginx._internal.configurator.subprocess.Popen") @@ -89,7 +89,7 @@ class NginxConfiguratorTest(util.NginxTest): "155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", - "headers.com", "example.net"}) + "headers.com", "example.net", "ssl.both.com"}) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -935,7 +935,19 @@ class NginxConfiguratorTest(util.NginxTest): prefer_ssl=False, no_ssl_filter_port='80') # Check that the dialog was called with only port 80 vhosts - self.assertEqual(len(mock_select_vhs.call_args[0][0]), 6) + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 8) + + def test_choose_auth_vhosts(self): + """choose_auth_vhosts correctly selects duplicative and HTTP/HTTPS vhosts""" + http, https = self.config.choose_auth_vhosts('ssl.both.com') + self.assertEqual(len(http), 4) + self.assertEqual(len(https), 2) + self.assertEqual(http[0].names, {'ssl.both.com'}) + self.assertEqual(http[1].names, {'ssl.both.com'}) + self.assertEqual(http[2].names, {'ssl.both.com'}) + self.assertEqual(http[3].names, {'*.both.com'}) + self.assertEqual(https[0].names, {'ssl.both.com'}) + self.assertEqual(https[1].names, {'*.both.com'}) class InstallSslOptionsConfTest(util.NginxTest): diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index 8f0673c1f..d0e84fc83 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -6,7 +6,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six from acme import challenges from certbot import achallenges @@ -45,6 +44,10 @@ class HttpPerformTest(util.NginxTest): challb=acme_util.chall_to_challb( challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), domain="migration.com", account_key=account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), + domain="ipv6ssl.com", account_key=account_key), ] def setUp(self): @@ -78,8 +81,8 @@ class HttpPerformTest(util.NginxTest): http_responses = self.http01.perform() - self.assertEqual(len(http_responses), 4) - for i in six.moves.range(4): + self.assertEqual(len(http_responses), 5) + for i in range(5): self.assertEqual(http_responses[i], acme_responses[i]) def test_mod_config(self): @@ -106,6 +109,43 @@ class HttpPerformTest(util.NginxTest): # self.assertEqual(vhost.addrs, set(v_addr2_print)) # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')])) + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_http_and_https(self, mock_add_server_directives): + """A server_name with both HTTP and HTTPS vhosts should get modded in both vhosts""" + self.configuration.https_port = 443 + self.http01.add_chall(self.achalls[3]) # migration.com + self.http01._mod_config() # pylint: disable=protected-access + + # Domain has an HTTP and HTTPS vhost + # 2 * 'rewrite' + 2 * 'return 200 keyauthz' = 4 + self.assertEqual(mock_add_server_directives.call_count, 4) + + @mock.patch('certbot_nginx._internal.parser.nginxparser.dump') + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_only_https(self, mock_add_server_directives, mock_dump): + """A server_name with only an HTTPS vhost should get modded""" + self.http01.add_chall(self.achalls[4]) # ipv6ssl.com + self.http01._mod_config() # pylint: disable=protected-access + + # It should modify the existing HTTPS vhost + self.assertEqual(mock_add_server_directives.call_count, 2) + # since there was no suitable HTTP vhost or default HTTP vhost, a non-empty one + # should have been created and written to the challenge conf file + self.assertNotEqual(mock_dump.call_args[0][0], []) + + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_deduplicate(self, mock_add_server_directives): + """A vhost that appears in both HTTP and HTTPS vhosts only gets modded once""" + achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), + domain="ssl.both.com", account_key=AUTH_KEY) + self.http01.add_chall(achall) + self.http01._mod_config() # pylint: disable=protected-access + + # Should only get called 5 times, rather than 6, because two vhosts are the same + self.assertEqual(mock_add_server_directives.call_count, 5*2) + @mock.patch("certbot_nginx._internal.configurator.NginxConfigurator.ipv6_info") def test_default_listen_addresses_no_memoization(self, ipv6_info): # pylint: disable=protected-access diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 0083c2448..35173d094 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -51,6 +51,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual({nparser.abs_path(x) for x in ['foo.conf', 'nginx.conf', 'server.conf', 'sites-enabled/default', + 'sites-enabled/both.com', 'sites-enabled/example.com', 'sites-enabled/headers.com', 'sites-enabled/migration.com', @@ -88,7 +89,7 @@ class NginxParserTest(util.NginxTest): parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(9, len( + self.assertEqual(10, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -111,7 +112,7 @@ class NginxParserTest(util.NginxTest): ([[[0], [3], [4]], [[5], [3], [0]]], [])] for mylist, result in mylists: - paths = [] # type: List[List[int]] + paths: List[List[int]] = [] parser._do_for_subarray(mylist, lambda x: isinstance(x, list) and len(x) >= 1 and @@ -171,7 +172,7 @@ class NginxParserTest(util.NginxTest): '*.www.example.com'}, [], [2, 1, 0]) - self.assertEqual(14, len(vhosts)) + self.assertEqual(19, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] diff --git a/certbot-nginx/tests/test_log_util.py b/certbot-nginx/tests/test_log_util.py deleted file mode 100644 index 7aebf2151..000000000 --- a/certbot-nginx/tests/test_log_util.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Backport for `TestCase.assertLogs()`. - -Most of the idea and code are from CPython implementation. -https://github.com/python/cpython/blob/b76518d43fb82ed9e5d27025d18c90a23d525c90/Lib/unittest/case.py -""" -import logging -import collections - -__all__ = ['AssertLogsMixin'] - -LoggingWatcher = collections.namedtuple('LoggingWatcher', ['records', 'output']) - - -class CapturingHandler(logging.Handler): - """ - A logging handler capturing all (raw and formatted) logging output. - """ - - def __init__(self): - super(CapturingHandler, self).__init__() - self.watcher = LoggingWatcher([], []) - - def flush(self): - pass - - def emit(self, record): - self.watcher.records.append(record) - self.watcher.output.append(self.format(record)) - - - -class AssertLogsContext(object): - """ - A context manager used to implement `TestCase.assertLogs()`. - """ - - LOGGING_FORMAT = '%(levelname)s:%(name)s:%(message)s' - - def __init__(self, test_case, logger_name, level): - self.test_case = test_case - - self.logger_name = logger_name - self.logger_states = None - self.logger = None - - if level: - # pylint: disable=protected-access,no-member - try: - # In Python 3.x - name_to_level = logging._nameToLevel # type: ignore - except AttributeError: - # In Python 2.7 - name_to_level = logging._levelNames # type: ignore - - self.level = name_to_level.get(level, level) - else: - self.level = logging.INFO - - self.watcher = None - - def _save_logger_states(self): - self.logger_states = (self.logger.handlers[:], self.logger.level, self.logger.propagate) - - def _restore_logger_states(self): - self.logger.handlers, self.logger.level, self.logger.propagate = self.logger_states - - def __enter__(self): - if isinstance(self.logger_name, logging.Logger): - self.logger = self.logger_name - else: - self.logger = logging.getLogger(self.logger_name) - - formatter = logging.Formatter(self.LOGGING_FORMAT) - - handler = CapturingHandler() - handler.setFormatter(formatter) - - self._save_logger_states() - self.logger.handlers = [handler] - self.logger.setLevel(self.level) - self.logger.propagate = False - - self.watcher = handler.watcher - return handler.watcher - - def __exit__(self, exc_type, exc_value, tb): - self._restore_logger_states() - - if exc_type is not None: - # let unexpected exceptions pass through - return - - if not self.watcher.records: - self._raiseFailure( - "no logs of level {} or higher triggered on {}" - .format(logging.getLevelName(self.level), self.logger.name)) - - def _raiseFailure(self, message): - message = self.test_case._formatMessage(None, message) # pylint: disable=protected-access - raise self.test_case.failureException(message) - - -class AssertLogsMixin(object): - """ - A mixin that implements `TestCase.assertLogs()`. - """ - - def assertLogs(self, logger=None, level=None): - """Fail unless a log message of level *level* or higher is emitted - on *logger_name* or its children. If omitted, *level* defaults to - INFO and *logger* defaults to the root logger. - This method must be used as a context manager, and will yield - a recording object with two attributes: `output` and `records`. - At the end of the context manager, the `output` attribute will - be a list of the matching formatted log messages and the - `records` attribute will be a list of the corresponding LogRecord - objects. - Example:: - with self.assertLogs('foo', level='INFO') as cm: - logging.getLogger('foo').info('first message') - logging.getLogger('foo.bar').error('second message') - self.assertEqual(cm.output, ['INFO:foo:first message', - 'ERROR:foo.bar:second message']) - """ - return AssertLogsContext(self, logger, level) diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index f545dc5bc..ee53e8e1e 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -17,10 +17,9 @@ from certbot.plugins import common from certbot.tests import util as test_util from certbot_nginx._internal import configurator from certbot_nginx._internal import nginxparser -import test_log_util -class NginxTest(test_log_util.AssertLogsMixin, test_util.ConfigTestCase): +class NginxTest(test_util.ConfigTestCase): def setUp(self): super(NginxTest, self).setUp() diff --git a/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com b/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com new file mode 100644 index 000000000..23a660df3 --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com @@ -0,0 +1,32 @@ +server { + server_name ssl.both.com; +} + +# a duplicate vhost +server { + server_name ssl.both.com; +} + +# a duplicate by means of wildcard +server { + server_name *.both.com; +} + +# combined HTTP and HTTPS +server { + server_name ssl.both.com; + listen 80; + listen 5001 ssl; + + ssl_certificate cert.pem; + ssl_certificate_key cert.key; +} + +# HTTPS, duplicate by means of wildcard +server { + server_name *.both.com; + listen 5001 ssl; + + ssl_certificate cert.pem; + ssl_certificate_key cert.key; +} diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 1333d2420..9b37e9a1a 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.13.0 - master +## 1.14.0 - master ### Added @@ -10,6 +10,28 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed +* certbot-auto no longer checks for updates on any operating system. +* The module `acme.magic_typing` is deprecated and will be removed in a future release. + Please use the built-in module `typing` instead. +* The DigitalOcean plugin now creates TXT records for the DNS-01 challenge with a lower 30s TTL. + +### Fixed + +* Don't output an empty line for a hidden certificate when `certbot certificates` is being used + in combination with `--cert-name` or `-d`. + +More details about these changes can be found on our GitHub repo. + +## 1.13.0 - 2021-03-02 + +### Added + +* + +### Changed + +* CLI flags `--os-packages-only`, `--no-self-upgrade`, `--no-bootstrap` and `--no-permissions-check`, + which are related to certbot-auto, are deprecated and will be removed in a future release. * Certbot no longer conditionally depends on an external mock module. Certbot's test API will continue to use it if it is available for backwards compatibility, however, this behavior has been deprecated and will be removed @@ -17,6 +39,12 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * The acme library no longer depends on the `security` extras from `requests` which was needed to support SNI in TLS requests when using old versions of Python 2. +* Certbot and all of its components no longer depend on the library `six`. +* The update of certbot-auto itself is now disabled on all RHEL-like systems. +* When revoking a certificate by `--cert-name`, it is no longer necessary to specify the `--server` + if the certificate was obtained from a non-default ACME server. +* The nginx authenticator now configures all matching HTTP and HTTPS vhosts for the HTTP-01 + challenge. It is now compatible with external HTTPS redirection by a CDN or load balancer. ### Fixed diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index be06b5803..790443bcc 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.13.0.dev0' +__version__ = '1.14.0.dev0' diff --git a/certbot/certbot/_internal/account.py b/certbot/certbot/_internal/account.py index b4619beba..c5667a865 100644 --- a/certbot/certbot/_internal/account.py +++ b/certbot/certbot/_internal/account.py @@ -10,7 +10,6 @@ from cryptography.hazmat.primitives import serialization import josepy as jose import pyrfc3339 import pytz -import six from acme import fields as acme_fields from acme import messages @@ -19,13 +18,13 @@ from certbot import errors from certbot import interfaces from certbot import util from certbot._internal import constants -from certbot.compat import os from certbot.compat import filesystem +from certbot.compat import os logger = logging.getLogger(__name__) -class Account(object): +class Account: """ACME protocol registration. :ivar .RegistrationResource regr: Registration Resource @@ -101,7 +100,7 @@ class AccountMemoryStorage(interfaces.AccountStorage): self.accounts = initial_accounts if initial_accounts is not None else {} def find_all(self): - return list(six.itervalues(self.accounts)) + return list(self.accounts.values()) def save(self, account, client): if account.id in self.accounts: @@ -230,8 +229,7 @@ class AccountFileStorage(interfaces.AccountStorage): def load(self, account_id): return self._load_for_server_path(account_id, self.config.server_path) - def save(self, account, client): - # type: (Account, ClientBase) -> None + def save(self, account: Account, client: ClientBase) -> None: """Create a new account. :param Account account: account to create @@ -246,8 +244,7 @@ class AccountFileStorage(interfaces.AccountStorage): except IOError as error: raise errors.AccountStorageError(error) - def update_regr(self, account, client): - # type: (Account, ClientBase) -> None + def update_regr(self, account: Account, client: ClientBase) -> None: """Update the registration resource. :param Account account: account to update @@ -260,8 +257,7 @@ class AccountFileStorage(interfaces.AccountStorage): except IOError as error: raise errors.AccountStorageError(error) - def update_meta(self, account): - # type: (Account) -> None + def update_meta(self, account: Account) -> None: """Update the meta resource. :param Account account: account to update @@ -339,19 +335,16 @@ class AccountFileStorage(interfaces.AccountStorage): return dir_path - def _prepare(self, account): - # type: (Account) -> str + def _prepare(self, account: Account) -> str: account_dir_path = self._account_dir_path(account.id) util.make_or_verify_dir(account_dir_path, 0o700, self.config.strict_permissions) return account_dir_path - def _create(self, account, dir_path): - # type: (Account, str) -> None + def _create(self, account: Account, dir_path: str) -> None: with util.safe_open(self._key_path(dir_path), "w", chmod=0o400) as key_file: key_file.write(account.key.json_dumps()) - def _update_regr(self, account, acme, dir_path): - # type: (Account, ClientBase, str) -> None + def _update_regr(self, account: Account, acme: ClientBase, dir_path: str) -> None: with open(self._regr_path(dir_path), "w") as regr_file: regr = account.regr # If we have a value for new-authz, save it for forwards diff --git a/certbot/certbot/_internal/auth_handler.py b/certbot/certbot/_internal/auth_handler.py index 7ea2a1de8..c2f323a36 100644 --- a/certbot/certbot/_internal/auth_handler.py +++ b/certbot/certbot/_internal/auth_handler.py @@ -2,15 +2,15 @@ import datetime import logging import time +from typing import Dict +from typing import List +from typing import Tuple import zope.component from acme import challenges from acme import errors as acme_errors from acme import messages -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Tuple from certbot import achallenges from certbot import errors from certbot import interfaces @@ -19,7 +19,7 @@ from certbot._internal import error_handler logger = logging.getLogger(__name__) -class AuthHandler(object): +class AuthHandler: """ACME Authorization Handler for a client. :ivar auth: Authenticator capable of solving @@ -98,8 +98,7 @@ class AuthHandler(object): return authzrs_validated - def deactivate_valid_authorizations(self, orderr): - # type: (messages.OrderResource) -> Tuple[List, List] + def deactivate_valid_authorizations(self, orderr: messages.OrderResource) -> Tuple[List, List]: """ Deactivate all `valid` authorizations in the order, so that they cannot be re-used in subsequent orders. @@ -191,7 +190,7 @@ class AuthHandler(object): """ pending_authzrs = [authzr for authzr in authzrs if authzr.body.status != messages.STATUS_VALID] - achalls = [] # type: List[achallenges.AnnotatedChallenge] + achalls: List[achallenges.AnnotatedChallenge] = [] if pending_authzrs: logger.info("Performing the following challenges:") for authzr in pending_authzrs: @@ -428,7 +427,7 @@ _ERROR_HELP = { def _report_failed_authzrs(failed_authzrs, account_key): """Notifies the user about failed authorizations.""" - problems = {} # type: Dict[str, List[achallenges.AnnotatedChallenge]] + problems: Dict[str, List[achallenges.AnnotatedChallenge]] = {} failed_achalls = [challb_to_achall(challb, account_key, authzr.body.identifier.value) for authzr in failed_authzrs for challb in authzr.body.challenges if challb.error] diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index dfbe4b538..b9b4ad2d7 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -3,11 +3,11 @@ import datetime import logging import re import traceback +from typing import List import pytz import zope.component -from acme.magic_typing import List from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -223,7 +223,7 @@ def cert_path_to_lineage(cli_config): """ acceptable_matches = _acceptable_matches() match = match_and_check_overlaps(cli_config, acceptable_matches, - lambda x: cli_config.cert_path[0], lambda x: x.lineagename) + lambda x: cli_config.cert_path, lambda x: x.lineagename) return match[0] @@ -241,7 +241,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func def find_matches(candidate_lineage, return_value, acceptable_matches): """Returns a list of matches using _search_lineages.""" acceptable_matches = [func(candidate_lineage) for func in acceptable_matches] - acceptable_matches_rv = [] # type: List[str] + acceptable_matches_rv: List[str] = [] for item in acceptable_matches: if isinstance(item, list): acceptable_matches_rv += item @@ -254,7 +254,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func matched = _search_lineages(cli_config, find_matches, [], acceptable_matches) if not matched: - raise errors.Error("No match found for cert-path {0}!".format(cli_config.cert_path[0])) + raise errors.Error("No match found for cert-path {0}!".format(cli_config.cert_path)) elif len(matched) > 1: raise errors.OverlappingMatchFound() return matched @@ -266,9 +266,9 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False): checker = ocsp.RevocationChecker() if config.certname and cert.lineagename != config.certname and not skip_filter_checks: - return "" + return None if config.domains and not set(config.domains).issubset(cert.names()): - return "" + return None now = pytz.UTC.fromutc(datetime.datetime.utcnow()) reasons = [] @@ -358,13 +358,15 @@ def _report_human_readable(config, parsed_certs): """Format a results report for a parsed cert""" certinfo = [] for cert in parsed_certs: - certinfo.append(human_readable_cert_info(config, cert)) + cert_info = human_readable_cert_info(config, cert) + if cert_info is not None: + certinfo.append(cert_info) return "\n".join(certinfo) def _describe_certs(config, parsed_certs, parse_failures): """Print information about the certs we know about""" - out = [] # type: List[str] + out: List[str] = [] notify = out.append diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index c69bb3564..d48fd419a 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -1,73 +1,58 @@ """Certbot command line argument & config processing.""" # pylint: disable=too-many-lines -from __future__ import print_function +import argparse import logging import logging.handlers -import argparse import sys -import certbot._internal.plugins.selection as plugin_selection -from certbot._internal.plugins import disco as plugins_disco +from typing import Optional -from acme.magic_typing import Optional - -# pylint: disable=ungrouped-imports import certbot from certbot._internal import constants - -import certbot.plugins.enhancements as enhancements - - -from certbot._internal.cli.cli_constants import ( - LEAUTO, - old_path_fragment, - new_path_prefix, - cli_command, - SHORT_USAGE, - COMMAND_OVERVIEW, - HELP_AND_VERSION_USAGE, - ARGPARSE_PARAMS_TO_REMOVE, - EXIT_ACTIONS, - ZERO_ARG_ACTIONS, - VAR_MODIFIERS, - DEPRECATED_OPTIONS -) - -from certbot._internal.cli.cli_utils import ( - _Default, - read_file, - flag_default, - config_help, - HelpfulArgumentGroup, - CustomHelpFormatter, - _DomainsAction, - add_domains, - CaseInsensitiveList, - _user_agent_comment_type, - _EncodeReasonAction, - parse_preferred_challenges, - _PrefChallAction, - _DeployHookAction, - _RenewHookAction, - nonnegative_int -) - -# These imports depend on cli_constants and cli_utils. -from certbot._internal.cli.verb_help import VERB_HELP, VERB_HELP_MAP +from certbot._internal.cli.cli_constants import ARGPARSE_PARAMS_TO_REMOVE +from certbot._internal.cli.cli_constants import cli_command +from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW +from certbot._internal.cli.cli_constants import DEPRECATED_OPTIONS +from certbot._internal.cli.cli_constants import EXIT_ACTIONS +from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE +from certbot._internal.cli.cli_constants import LEAUTO +from certbot._internal.cli.cli_constants import new_path_prefix +from certbot._internal.cli.cli_constants import old_path_fragment +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_constants import VAR_MODIFIERS +from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS +from certbot._internal.cli.cli_utils import _Default +from certbot._internal.cli.cli_utils import _DeployHookAction +from certbot._internal.cli.cli_utils import _DomainsAction +from certbot._internal.cli.cli_utils import _EncodeReasonAction +from certbot._internal.cli.cli_utils import _PrefChallAction +from certbot._internal.cli.cli_utils import _RenewHookAction +from certbot._internal.cli.cli_utils import _user_agent_comment_type +from certbot._internal.cli.cli_utils import add_domains +from certbot._internal.cli.cli_utils import CaseInsensitiveList +from certbot._internal.cli.cli_utils import config_help +from certbot._internal.cli.cli_utils import CustomHelpFormatter +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import HelpfulArgumentGroup +from certbot._internal.cli.cli_utils import nonnegative_int +from certbot._internal.cli.cli_utils import parse_preferred_challenges +from certbot._internal.cli.cli_utils import read_file from certbot._internal.cli.group_adder import _add_all_groups -from certbot._internal.cli.subparsers import _create_subparsers +from certbot._internal.cli.helpful import HelpfulArgumentParser from certbot._internal.cli.paths_parser import _paths_parser from certbot._internal.cli.plugins_parsing import _plugins_parsing - -# These imports depend on some or all of the submodules for cli. -from certbot._internal.cli.helpful import HelpfulArgumentParser -# pylint: enable=ungrouped-imports +from certbot._internal.cli.subparsers import _create_subparsers +from certbot._internal.cli.verb_help import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP_MAP +from certbot._internal.plugins import disco as plugins_disco +import certbot._internal.plugins.selection as plugin_selection +import certbot.plugins.enhancements as enhancements logger = logging.getLogger(__name__) # Global, to save us from a lot of argument passing within the scope of this module -helpful_parser = None # type: Optional[HelpfulArgumentParser] +helpful_parser: Optional[HelpfulArgumentParser] = None def prepare_and_parse_args(plugins, args, detect_defaults=False): @@ -249,27 +234,6 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): default=flag_default("duplicate"), help="Allow making a certificate lineage that duplicates an existing one " "(both can be renewed in parallel)") - helpful.add( - "automation", "--os-packages-only", action="store_true", - default=flag_default("os_packages_only"), - help="(certbot-auto only) install OS package dependencies and then stop") - helpful.add( - "automation", "--no-self-upgrade", action="store_true", - default=flag_default("no_self_upgrade"), - help="(certbot-auto only) prevent the certbot-auto script from" - " upgrading itself to newer released versions (default: Upgrade" - " automatically)") - helpful.add( - "automation", "--no-bootstrap", action="store_true", - default=flag_default("no_bootstrap"), - help="(certbot-auto only) prevent the certbot-auto script from" - " installing OS-level dependencies (default: Prompt to install " - " OS-wide dependencies, but exit if the user says 'No')") - helpful.add( - "automation", "--no-permissions-check", action="store_true", - default=flag_default("no_permissions_check"), - help="(certbot-auto only) skip the check on the file system" - " permissions of the certbot-auto script") helpful.add( ["automation", "renew", "certonly", "run"], "-q", "--quiet", dest="quiet", action="store_true", @@ -451,6 +415,12 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): default=flag_default("autorenew"), dest="autorenew", help="Disable auto renewal of certificates.") + # Deprecated arguments + helpful.add_deprecated_argument("--os-packages-only", 0) + helpful.add_deprecated_argument("--no-self-upgrade", 0) + helpful.add_deprecated_argument("--no-bootstrap", 0) + helpful.add_deprecated_argument("--no-permissions-check", 0) + # Populate the command line parameters for new style enhancements enhancements.populate_cli(helpful.add) diff --git a/certbot/certbot/_internal/cli/cli_constants.py b/certbot/certbot/_internal/cli/cli_constants.py index dc199e152..bd25f9bee 100644 --- a/certbot/certbot/_internal/cli/cli_constants.py +++ b/certbot/certbot/_internal/cli/cli_constants.py @@ -109,4 +109,10 @@ VAR_MODIFIERS = {"account": {"server",}, # This is a list of all CLI options that we have ever deprecated. It lets us # opt out of the default detection, which can interact strangely with option # deprecation. See https://github.com/certbot/certbot/issues/8540 for more info. -DEPRECATED_OPTIONS = {"manual_public_ip_logging_ok",} +DEPRECATED_OPTIONS = { + "manual_public_ip_logging_ok", + "os_packages_only", + "no_self_upgrade", + "no_bootstrap", + "no_permissions_check", +} diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index a0ddce38f..1f30261c1 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -5,14 +5,14 @@ import copy import zope.interface.interface # pylint: disable=unused-import from acme import challenges +from certbot import errors from certbot import interfaces from certbot import util -from certbot import errors -from certbot.compat import os from certbot._internal import constants +from certbot.compat import os -class _Default(object): +class _Default: """A class to use as a default to detect if a value is set by a user""" def __bool__(self): @@ -62,11 +62,11 @@ def config_help(name, hidden=False): """Extract the help message for an `.IConfig` attribute.""" if hidden: return argparse.SUPPRESS - field = interfaces.IConfig.__getitem__(name) # type: zope.interface.interface.Attribute + field: zope.interface.interface.Attribute = interfaces.IConfig.__getitem__(name) return field.__doc__ -class HelpfulArgumentGroup(object): +class HelpfulArgumentGroup: """Emulates an argparse group for use with HelpfulArgumentParser. This class is used in the add_group method of HelpfulArgumentParser. diff --git a/certbot/certbot/_internal/cli/group_adder.py b/certbot/certbot/_internal/cli/group_adder.py index f22fbc496..0c54c9fe1 100644 --- a/certbot/certbot/_internal/cli/group_adder.py +++ b/certbot/certbot/_internal/cli/group_adder.py @@ -1,6 +1,6 @@ """This module contains a function to add the groups of arguments for the help display""" -from certbot._internal.cli import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP def _add_all_groups(helpful): diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index 2cb506157..f185bdc26 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -1,48 +1,42 @@ """Certbot command line argument parser""" -from __future__ import print_function + import argparse import copy import functools import glob import sys +from typing import Any +from typing import Dict import configargparse -import six import zope.component import zope.interface - from zope.interface import interfaces as zope_interfaces -from acme.magic_typing import Any, Dict - from certbot import crypto_util from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import os from certbot._internal import constants from certbot._internal import hooks - +from certbot._internal.cli.cli_constants import ARGPARSE_PARAMS_TO_REMOVE +from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW +from certbot._internal.cli.cli_constants import EXIT_ACTIONS +from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS +from certbot._internal.cli.cli_utils import _Default +from certbot._internal.cli.cli_utils import add_domains +from certbot._internal.cli.cli_utils import CustomHelpFormatter +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import HelpfulArgumentGroup +from certbot._internal.cli.verb_help import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP_MAP +from certbot.compat import os from certbot.display import util as display_util -from certbot._internal.cli import ( - SHORT_USAGE, - CustomHelpFormatter, - flag_default, - VERB_HELP, - VERB_HELP_MAP, - COMMAND_OVERVIEW, - HELP_AND_VERSION_USAGE, - _Default, - add_domains, - EXIT_ACTIONS, - ZERO_ARG_ACTIONS, - ARGPARSE_PARAMS_TO_REMOVE, - HelpfulArgumentGroup -) - -class HelpfulArgumentParser(object): +class HelpfulArgumentParser: """Argparse Wrapper. This class wraps argparse, adding the ability to make --help less @@ -99,16 +93,16 @@ class HelpfulArgumentParser(object): if isinstance(help1, bool) and isinstance(help2, bool): self.help_arg = help1 or help2 else: - self.help_arg = help1 if isinstance(help1, six.string_types) else help2 + self.help_arg = help1 if isinstance(help1, str) else help2 short_usage = self._usage_string(plugins, self.help_arg) self.visible_topics = self.determine_help_topics(self.help_arg) # elements are added by .add_group() - self.groups = {} # type: Dict[str, argparse._ArgumentGroup] + self.groups: Dict[str, argparse._ArgumentGroup] = {} # elements are added by .parse_args() - self.defaults = {} # type: Dict[str, Any] + self.defaults: Dict[str, Any] = {} self.parser = configargparse.ArgParser( prog="certbot", @@ -470,7 +464,7 @@ class HelpfulArgumentParser(object): may or may not be displayed as help topics. """ - for name, plugin_ep in six.iteritems(plugins): + for name, plugin_ep in plugins.items(): parser_or_group = self.add_group(name, description=plugin_ep.long_description) plugin_ep.plugin_cls.inject_parser_options(parser_or_group, name) diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py index 62f5e224d..6197a4bf9 100644 --- a/certbot/certbot/_internal/cli/paths_parser.py +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -1,11 +1,8 @@ """This is a module that adds configuration to the argument parser regarding paths for certificates""" +from certbot._internal.cli.cli_utils import config_help +from certbot._internal.cli.cli_utils import flag_default from certbot.compat import os -from certbot._internal.cli import ( - read_file, - flag_default, - config_help -) def _paths_parser(helpful): @@ -14,22 +11,19 @@ def _paths_parser(helpful): if verb == "help": verb = helpful.help_arg - cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked." - sections = ["paths", "install", "revoke", "certonly", "manage"] + cpkwargs = { + "type": os.path.abspath, + "help": "Path to where certificate is saved (with certonly --csr), installed " + "from, or revoked" + } if verb == "certonly": - add(sections, "--cert-path", type=os.path.abspath, - default=flag_default("auth_cert_path"), help=cph) - elif verb == "revoke": - add(sections, "--cert-path", type=read_file, required=False, help=cph) - else: - add(sections, "--cert-path", type=os.path.abspath, help=cph) + cpkwargs["default"] = flag_default("auth_cert_path") + add(["paths", "install", "revoke", "certonly", "manage"], "--cert-path", **cpkwargs) section = "paths" if verb in ("install", "revoke"): section = verb - # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", - type=((verb == "revoke" and read_file) or os.path.abspath), + add(section, "--key-path", type=os.path.abspath, help="Path to private key for certificate installation " "or revocation (if account key is missing)") diff --git a/certbot/certbot/_internal/cli/plugins_parsing.py b/certbot/certbot/_internal/cli/plugins_parsing.py index 9e11ad3ab..bbfdf22da 100644 --- a/certbot/certbot/_internal/cli/plugins_parsing.py +++ b/certbot/certbot/_internal/cli/plugins_parsing.py @@ -1,5 +1,5 @@ """This is a module that handles parsing of plugins for the argument parser""" -from certbot._internal.cli import flag_default +from certbot._internal.cli.cli_utils import flag_default def _plugins_parsing(helpful, plugins): diff --git a/certbot/certbot/_internal/cli/subparsers.py b/certbot/certbot/_internal/cli/subparsers.py index 13f8705ce..822381d21 100644 --- a/certbot/certbot/_internal/cli/subparsers.py +++ b/certbot/certbot/_internal/cli/subparsers.py @@ -1,14 +1,11 @@ """This module creates subparsers for the argument parser""" from certbot import interfaces from certbot._internal import constants - -from certbot._internal.cli import ( - flag_default, - read_file, - CaseInsensitiveList, - _user_agent_comment_type, - _EncodeReasonAction -) +from certbot._internal.cli.cli_utils import _EncodeReasonAction +from certbot._internal.cli.cli_utils import _user_agent_comment_type +from certbot._internal.cli.cli_utils import CaseInsensitiveList +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import read_file def _create_subparsers(helpful): diff --git a/certbot/certbot/_internal/cli/verb_help.py b/certbot/certbot/_internal/cli/verb_help.py index 131cfec96..dfb4e9c67 100644 --- a/certbot/certbot/_internal/cli/verb_help.py +++ b/certbot/certbot/_internal/cli/verb_help.py @@ -1,9 +1,7 @@ """This module contain help information for verbs supported by certbot""" +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_utils import flag_default from certbot.compat import os -from certbot._internal.cli import ( - SHORT_USAGE, - flag_default -) # The attributes here are: # short: a string that will be displayed by "certbot -h commands" diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index 7ebee72eb..ae24276ae 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -2,6 +2,8 @@ import datetime import logging import platform +from typing import List +from typing import Optional from cryptography.hazmat.backends import default_backend # See https://github.com/pyca/cryptography/issues/4275 @@ -14,8 +16,6 @@ from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors from acme import messages -from acme.magic_typing import List -from acme.magic_typing import Optional import certbot from certbot import crypto_util from certbot import errors @@ -93,7 +93,7 @@ def ua_flags(config): flags.append("hook") return " ".join(flags) -class DummyConfig(object): +class DummyConfig: "Shim for computing a sample user agent." def __init__(self): self.authenticator = "XXX" @@ -227,7 +227,7 @@ def perform_registration(acme, config, tos_cb): raise -class Client(object): +class Client: """Certbot's client. :ivar .IConfig config: Client configuration. @@ -328,7 +328,7 @@ class Client(object): with open(old_keypath, "rb") as f: keypath = old_keypath keypem = f.read() - key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key] + key: Optional[util.Key] = util.Key(file=keypath, pem=keypem) logger.info("Reusing existing private key from %s.", old_keypath) else: # The key is set to None here but will be created below. @@ -390,8 +390,8 @@ class Client(object): cert, chain = self.obtain_certificate_from_csr(csr, orderr) return cert, chain, key, csr - def _get_order_and_authorizations(self, csr_pem, best_effort): - # type: (str, bool) -> List[messages.OrderResource] + def _get_order_and_authorizations(self, csr_pem: str, + best_effort: bool) -> List[messages.OrderResource]: """Request a new order and complete its authorizations. :param str csr_pem: A CSR in PEM format. diff --git a/certbot/certbot/_internal/configuration.py b/certbot/certbot/_internal/configuration.py index f1e85f9fe..aee0022b8 100644 --- a/certbot/certbot/_internal/configuration.py +++ b/certbot/certbot/_internal/configuration.py @@ -1,7 +1,7 @@ """Certbot user-supplied configuration.""" import copy +from urllib import parse -from six.moves.urllib import parse import zope.interface from certbot import errors @@ -13,7 +13,7 @@ from certbot.compat import os @zope.interface.implementer(interfaces.IConfig) -class NamespaceConfig(object): +class NamespaceConfig: """Configuration wrapper around :class:`argparse.Namespace`. For more documentation, including available attributes, please see diff --git a/certbot/certbot/_internal/display/completer.py b/certbot/certbot/_internal/display/completer.py index 03719862b..a6c984195 100644 --- a/certbot/certbot/_internal/display/completer.py +++ b/certbot/certbot/_internal/display/completer.py @@ -8,7 +8,7 @@ except ImportError: import certbot._internal.display.dummy_readline as readline # type: ignore -class Completer(object): +class Completer: """Provides Tab completion when prompting users for a path. This class is meant to be used with readline to provide Tab diff --git a/certbot/certbot/_internal/eff.py b/certbot/certbot/_internal/eff.py index 5fbbd302a..b01e2dd61 100644 --- a/certbot/certbot/_internal/eff.py +++ b/certbot/certbot/_internal/eff.py @@ -1,23 +1,21 @@ """Subscribes users to the EFF newsletter.""" import logging +from typing import Optional import requests import zope.component -from acme.magic_typing import Optional # pylint: disable=unused-import - from certbot import interfaces -from certbot.display import util as display_util from certbot._internal import constants -from certbot._internal.account import Account # pylint: disable=unused-import +from certbot._internal.account import Account from certbot._internal.account import AccountFileStorage -from certbot.interfaces import IConfig # pylint: disable=unused-import +from certbot.display import util as display_util +from certbot.interfaces import IConfig logger = logging.getLogger(__name__) -def prepare_subscription(config, acc): - # type: (IConfig, Account) -> None +def prepare_subscription(config: IConfig, acc: Account) -> None: """High level function to store potential EFF newsletter subscriptions. The user may be asked if they want to sign up for the newsletter if @@ -45,8 +43,7 @@ def prepare_subscription(config, acc): storage.update_meta(acc) -def handle_subscription(config, acc): - # type: (IConfig, Account) -> None +def handle_subscription(config: IConfig, acc: Account) -> None: """High level function to take care of EFF newsletter subscriptions. Once subscription is handled, it will not be handled again. @@ -65,8 +62,7 @@ def handle_subscription(config, acc): storage.update_meta(acc) -def _want_subscription(): - # type: () -> bool +def _want_subscription() -> bool: """Does the user want to be subscribed to the EFF newsletter? :returns: True if we should subscribe the user, otherwise, False @@ -83,8 +79,7 @@ def _want_subscription(): return display.yesno(prompt, default=False) -def subscribe(email): - # type: (str) -> None +def subscribe(email: str) -> None: """Subscribe the user to the EFF mailing list. :param str email: the e-mail address to subscribe @@ -99,8 +94,7 @@ def subscribe(email): _check_response(requests.post(url, data=data)) -def _check_response(response): - # type: (requests.Response) -> None +def _check_response(response: requests.Response) -> None: """Check for errors in the server's response. If an error occurred, it will be reported to the user. @@ -120,8 +114,7 @@ def _check_response(response): _report_failure('there was a problem with the server response') -def _report_failure(reason=None): - # type: (Optional[str]) -> None +def _report_failure(reason: Optional[str] = None) -> None: """Notify the user of failing to sign them up for the newsletter. :param reason: a phrase describing what the problem was diff --git a/certbot/certbot/_internal/error_handler.py b/certbot/certbot/_internal/error_handler.py index 60fb287a6..01cc92b42 100644 --- a/certbot/certbot/_internal/error_handler.py +++ b/certbot/certbot/_internal/error_handler.py @@ -3,12 +3,12 @@ import functools import logging import signal import traceback +from typing import Any +from typing import Callable +from typing import Dict +from typing import List +from typing import Union -from acme.magic_typing import Any -from acme.magic_typing import Callable -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Union from certbot import errors from certbot.compat import os @@ -45,7 +45,7 @@ else: _SIGNALS = [] -class ErrorHandler(object): +class ErrorHandler: """Context manager for running code that must be cleaned up on failure. The context manager allows you to register functions that will be called @@ -77,9 +77,9 @@ class ErrorHandler(object): def __init__(self, func, *args, **kwargs): self.call_on_regular_exit = False self.body_executed = False - self.funcs = [] # type: List[Callable[[], Any]] - self.prev_handlers = {} # type: Dict[int, Union[int, None, Callable]] - self.received_signals = [] # type: List[int] + self.funcs: List[Callable[[], Any]] = [] + self.prev_handlers: Dict[int, Union[int, None, Callable]] = {} + self.received_signals: List[int] = [] if func is not None: self.register(func, *args, **kwargs) @@ -108,8 +108,7 @@ class ErrorHandler(object): self._call_signals() return retval - def register(self, func, *args, **kwargs): - # type: (Callable, *Any, **Any) -> None + def register(self, func: Callable, *args: Any, **kwargs: Any) -> None: """Sets func to be run with the given arguments during cleanup. :param function func: function to be called in case of an error diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index 5526b21c4..b9f1f1531 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -1,10 +1,9 @@ """Facilities for implementing hooks that call shell commands.""" -from __future__ import print_function import logging +from typing import List +from typing import Set -from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot import util from certbot.compat import filesystem @@ -78,7 +77,7 @@ def pre_hook(config): _run_pre_hook_if_necessary(cmd) -executed_pre_hooks = set() # type: Set[str] +executed_pre_hooks: Set[str] = set() def _run_pre_hook_if_necessary(command): @@ -128,7 +127,7 @@ def post_hook(config): _run_hook("post-hook", cmd) -post_hooks = [] # type: List[str] +post_hooks: List[str] = [] def _run_eventually(command): diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index aa0b80eaa..d00302598 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -1,8 +1,8 @@ """Implements file locks compatible with Linux and Windows for locking files and directories.""" import errno import logging +from typing import Optional -from acme.magic_typing import Optional from certbot import errors from certbot.compat import filesystem from certbot.compat import os @@ -16,29 +16,10 @@ else: POSIX_MODE = True - logger = logging.getLogger(__name__) -def lock_dir(dir_path): - # type: (str) -> LockFile - """Place a lock file on the directory at dir_path. - - The lock file is placed in the root of dir_path with the name - .certbot.lock. - - :param str dir_path: path to directory - - :returns: the locked LockFile object - :rtype: LockFile - - :raises errors.LockError: if unable to acquire the lock - - """ - return LockFile(os.path.join(dir_path, '.certbot.lock')) - - -class LockFile(object): +class LockFile: """ Platform independent file lock system. LockFile accepts a parameter, the path to a file acting as a lock. Once the LockFile, @@ -52,8 +33,7 @@ class LockFile(object): LockFile is platform independent: it will proceed to the appropriate OS lock mechanism depending on Linux or Windows. """ - def __init__(self, path): - # type: (str) -> None + def __init__(self, path: str) -> None: """ Create a LockFile instance on the given file path, and acquire lock. :param str path: the path to the file that will hold a lock @@ -64,8 +44,7 @@ class LockFile(object): self.acquire() - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) if self.is_locked(): repr_str += 'acquired>' @@ -73,23 +52,20 @@ class LockFile(object): repr_str += 'released>' return repr_str - def acquire(self): - # type: () -> None + def acquire(self) -> None: """ Acquire the lock on the file, forbidding any other Certbot instance to acquire it. :raises errors.LockError: if unable to acquire the lock """ self._lock_mechanism.acquire() - def release(self): - # type: () -> None + def release(self) -> None: """ Release the lock on the file, allowing any other Certbot instance to acquire it. """ self._lock_mechanism.release() - def is_locked(self): - # type: () -> bool + def is_locked(self) -> bool: """ Check if the file is currently locked. :return: True if the file is locked, False otherwise @@ -97,18 +73,16 @@ class LockFile(object): return self._lock_mechanism.is_locked() -class _BaseLockMechanism(object): - def __init__(self, path): - # type: (str) -> None +class _BaseLockMechanism: + def __init__(self, path: str) -> None: """ Create a lock file mechanism for Unix. :param str path: the path to the lock file """ self._path = path - self._fd = None # type: Optional[int] + self._fd: Optional[int] = None - def is_locked(self): - # type: () -> bool + def is_locked(self) -> bool: """Check if lock file is currently locked. :return: True if the lock file is locked :rtype: bool @@ -129,8 +103,7 @@ class _UnixLockMechanism(_BaseLockMechanism): process exits. It cannot be used to provide synchronization between threads. It is based on the lock_file package by Martin Horcicka. """ - def acquire(self): - # type: () -> None + def acquire(self) -> None: """Acquire the lock.""" while self._fd is None: # Open the file @@ -144,8 +117,7 @@ class _UnixLockMechanism(_BaseLockMechanism): if self._fd is None: os.close(fd) - def _try_lock(self, fd): - # type: (int) -> None + def _try_lock(self, fd: int) -> None: """ Try to acquire the lock file without blocking. :param int fd: file descriptor of the opened file to lock @@ -158,8 +130,7 @@ class _UnixLockMechanism(_BaseLockMechanism): raise errors.LockError('Another instance of Certbot is already running.') raise - def _lock_success(self, fd): - # type: (int) -> bool + def _lock_success(self, fd: int) -> bool: """ Did we successfully grab the lock? Because this class deletes the locked file when the lock is @@ -185,8 +156,7 @@ class _UnixLockMechanism(_BaseLockMechanism): # the same device and inode, they're the same file. return stat1.st_dev == stat2.st_dev and stat1.st_ino == stat2.st_ino - def release(self): - # type: () -> None + def release(self) -> None: """Remove, close, and release the lock file.""" # It is important the lock file is removed before it's released, # otherwise: @@ -269,3 +239,20 @@ class _WindowsLockMechanism(_BaseLockMechanism): logger.debug(str(e)) finally: self._fd = None + + +def lock_dir(dir_path: str) -> LockFile: + """Place a lock file on the directory at dir_path. + + The lock file is placed in the root of dir_path with the name + .certbot.lock. + + :param str dir_path: path to directory + + :returns: the locked LockFile object + :rtype: LockFile + + :raises errors.LockError: if unable to acquire the lock + + """ + return LockFile(os.path.join(dir_path, '.certbot.lock')) diff --git a/certbot/certbot/_internal/log.py b/certbot/certbot/_internal/log.py index 90dc2cda1..d8a55e27d 100644 --- a/certbot/certbot/_internal/log.py +++ b/certbot/certbot/_internal/log.py @@ -19,7 +19,7 @@ The preferred method to display important information to the user is to use `certbot.display.util` and `certbot.display.ops`. """ -from __future__ import print_function + import functools import logging diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index e77c56ac2..8240bad20 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -1,17 +1,20 @@ """Certbot main entry point.""" # pylint: disable=too-many-lines -from __future__ import print_function import functools import logging.handlers import sys +from typing import Iterable +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union import configobj import josepy as jose import zope.component from acme import errors as acme_errors -from acme.magic_typing import Union, Iterable, Optional, List, Tuple # pylint: disable=unused-import import certbot from certbot import crypto_util from certbot import errors @@ -127,8 +130,8 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N return lineage -def _handle_unexpected_key_type_migration(config, cert): - # type: (configuration.NamespaceConfig, storage.RenewableCert) -> None +def _handle_unexpected_key_type_migration(config: configuration.NamespaceConfig, + cert: storage.RenewableCert) -> None: """ This function ensures that the user will not implicitly migrate an existing key from one type to another in the situation where a certificate for that lineage @@ -149,11 +152,10 @@ def _handle_unexpected_key_type_migration(config, cert): raise errors.Error(msg) -def _handle_subset_cert_request(config, # type: configuration.NamespaceConfig - domains, # type: List[str] - cert # type: storage.RenewableCert - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _handle_subset_cert_request(config: configuration.NamespaceConfig, + domains: List[str], + cert: storage.RenewableCert + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Figure out what to do if a previous cert had a subset of the names now requested :param config: Configuration object @@ -200,10 +202,9 @@ def _handle_subset_cert_request(config, # type: configuration.NamespaceConfig raise errors.Error(USER_CANCELLED) -def _handle_identical_cert_request(config, # type: configuration.NamespaceConfig - lineage, # type: storage.RenewableCert - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _handle_identical_cert_request(config: configuration.NamespaceConfig, + lineage: storage.RenewableCert, + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Figure out what to do if a lineage has the same names as a previously obtained one :param config: Configuration object @@ -319,11 +320,10 @@ def _find_cert(config, domains, certname): return (action != "reinstall"), lineage -def _find_lineage_for_domains_and_certname(config, # type: configuration.NamespaceConfig - domains, # type: List[str] - certname # type: str - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _find_lineage_for_domains_and_certname(config: configuration.NamespaceConfig, + domains: List[str], + certname: str + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Find appropriate lineage based on given domains and/or certname. :param config: Configuration object @@ -782,7 +782,7 @@ def update_account(config, unused_plugins): cb_client = client.Client(config, acc, None, None, acme=acme) # Empty list of contacts in case the user is removing all emails - acc_contacts = () # type: Iterable[str] + acc_contacts: Iterable[str] = () if config.email: acc_contacts = ['mailto:' + email for email in config.email.split(',')] # We rely on an exception to interrupt this process if it didn't work. @@ -1099,8 +1099,7 @@ def certificates(config, unused_plugins): cert_manager.certificates(config) -# TODO: coop with renewal config -def revoke(config, unused_plugins): +def revoke(config, unused_plugins: plugins_disco.PluginsRegistry) -> Optional[str]: """Revoke a previously obtained certificate. :param config: Configuration object @@ -1117,7 +1116,13 @@ def revoke(config, unused_plugins): config.installer = config.authenticator = None if config.cert_path is None and config.certname: - config.cert_path = storage.cert_path_for_cert_name(config, config.certname) + # When revoking via --cert-name, take the cert path and server from renewalparams + lineage = storage.RenewableCert( + storage.renewal_file_for_certname(config, config.certname), config) + config.cert_path = lineage.cert_path + # --server takes priority over lineage.server + if lineage.server and not cli.set_by_cli("server"): + config.server = lineage.server elif not config.cert_path or (config.cert_path and config.certname): # intentionally not supporting --cert-path & --cert-name together, # to avoid dealing with mismatched values @@ -1125,15 +1130,18 @@ def revoke(config, unused_plugins): if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using certificate key %s", - config.cert_path[0], config.key_path[0]) - crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) - key = jose.JWK.load(config.key_path[1]) + config.cert_path, config.key_path) + crypto_util.verify_cert_matches_priv_key(config.cert_path, config.key_path) + with open(config.key_path, 'rb') as f: + key = jose.JWK.load(f.read()) acme = client.acme_from_config_key(config, key) else: # revocation by account key - logger.debug("Revoking %s using Account Key", config.cert_path[0]) + logger.debug("Revoking %s using Account Key", config.cert_path) acc, _ = _determine_account(config) acme = client.acme_from_config_key(config, acc.key, acc.regr) - cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] + + with open(config.cert_path, 'rb') as f: + cert = crypto_util.pyopenssl_load_certificate(f.read())[0] logger.debug("Reason code for revocation: %s", config.reason) try: acme.revoke(jose.ComparableX509(cert), config.reason) @@ -1141,7 +1149,7 @@ def revoke(config, unused_plugins): except acme_errors.ClientError as e: return str(e) - display_ops.success_revocation(config.cert_path[0]) + display_ops.success_revocation(config.cert_path) return None @@ -1388,8 +1396,8 @@ def set_displayer(config): """ if config.quiet: config.noninteractive_mode = True - displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) \ - # type: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] + displayer: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] =\ + display_util.NoninteractiveDisplay(open(os.devnull, "w")) elif config.noninteractive_mode: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index bed22b4ae..4097544e8 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -1,26 +1,19 @@ """Utilities for plugins discovery and selection.""" -import collections +from collections.abc import Mapping import itertools import logging import sys +from typing import Dict import pkg_resources -import six import zope.interface import zope.interface.verify -from acme.magic_typing import Dict from certbot import errors from certbot import interfaces from certbot._internal import constants from certbot.compat import os -try: - # Python 3.3+ - from collections.abc import Mapping -except ImportError: # pragma: no cover - from collections import Mapping - logger = logging.getLogger(__name__) PREFIX_FREE_DISTRIBUTIONS = [ @@ -45,7 +38,7 @@ PREFIX_FREE_DISTRIBUTIONS = [ """Distributions for which prefix will be omitted.""" -class PluginEntryPoint(object): +class PluginEntryPoint: """Plugin entry point.""" # this object is mutable, don't allow it to be hashed! @@ -215,12 +208,12 @@ class PluginsRegistry(Mapping): # This prevents deadlock caused by plugins acquiring a lock # and ensures at least one concurrent Certbot instance will run # successfully. - self._plugins = collections.OrderedDict(sorted(six.iteritems(plugins))) + self._plugins = dict(sorted(plugins.items())) @classmethod def find_all(cls): """Find plugins using setuptools entry points.""" - plugins = {} # type: Dict[str, PluginEntryPoint] + plugins: Dict[str, PluginEntryPoint] = {} plugin_paths_string = os.getenv('CERTBOT_PLUGIN_PATH') plugin_paths = plugin_paths_string.split(':') if plugin_paths_string else [] # XXX should ensure this only happens once @@ -276,12 +269,12 @@ class PluginsRegistry(Mapping): def init(self, config): """Initialize all plugins in the registry.""" return [plugin_ep.init(config) for plugin_ep - in six.itervalues(self._plugins)] + in self._plugins.values()] def filter(self, pred): """Filter plugins based on predicate.""" return type(self)({name: plugin_ep for name, plugin_ep - in six.iteritems(self._plugins) if pred(plugin_ep)}) + in self._plugins.items() if pred(plugin_ep)}) def visible(self): """Filter plugins based on visibility.""" @@ -297,7 +290,7 @@ class PluginsRegistry(Mapping): def prepare(self): """Prepare all plugins in the registry.""" - return [plugin_ep.prepare() for plugin_ep in six.itervalues(self._plugins)] + return [plugin_ep.prepare() for plugin_ep in self._plugins.values()] def available(self): """Filter plugins based on availability.""" @@ -319,7 +312,7 @@ class PluginsRegistry(Mapping): """ # use list instead of set because PluginEntryPoint is not hashable - candidates = [plugin_ep for plugin_ep in six.itervalues(self._plugins) + candidates = [plugin_ep for plugin_ep in self._plugins.values() if plugin_ep.initialized and plugin_ep.init() is plugin] assert len(candidates) <= 1 if candidates: @@ -329,9 +322,9 @@ class PluginsRegistry(Mapping): def __repr__(self): return "{0}({1})".format( self.__class__.__name__, ','.join( - repr(p_ep) for p_ep in six.itervalues(self._plugins))) + repr(p_ep) for p_ep in self._plugins.values())) def __str__(self): if not self._plugins: return "No plugins" - return "\n\n".join(str(p_ep) for p_ep in six.itervalues(self._plugins)) + return "\n\n".join(str(p_ep) for p_ep in self._plugins.values()) diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index a2e4bb28e..ed2e0559e 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -1,10 +1,11 @@ """Manual authenticator plugin""" +from typing import Dict + import zope.component import zope.interface from acme import challenges -from acme.magic_typing import Dict -from certbot import achallenges # pylint: disable=unused-import +from certbot import achallenges from certbot import errors from certbot import interfaces from certbot import reverter @@ -73,8 +74,7 @@ permitted by DNS standards.) super(Authenticator, self).__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() - self.env = {} \ - # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] + self.env: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] = {} self.subsequent_dns_challenge = False self.subsequent_any_challenge = False diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index 0b04791c6..bf7e505b9 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -1,9 +1,7 @@ """Decide which plugins to use for authentication & installation""" -from __future__ import print_function import logging -import six import zope.component from certbot import errors @@ -108,7 +106,7 @@ def pick_plugin(config, default, plugins, question, ifaces): if len(prepared) > 1: logger.debug("Multiple candidate plugins: %s", prepared) - plugin_ep = choose_plugin(list(six.itervalues(prepared)), question) + plugin_ep = choose_plugin(list(prepared.values()), question) if plugin_ep is None: return None return plugin_ep.init() diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index bbb56178c..833651346 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -4,18 +4,17 @@ import logging import socket # https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi from socket import errno as socket_errors # type: ignore +from typing import DefaultDict +from typing import Dict +from typing import Set +from typing import Tuple +from typing import TYPE_CHECKING -import OpenSSL # pylint: disable=unused-import -import six +import OpenSSL import zope.interface from acme import challenges from acme import standalone as acme_standalone -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import Set -from acme.magic_typing import Tuple -from acme.magic_typing import TYPE_CHECKING from certbot import achallenges from certbot import errors from certbot import interfaces @@ -29,7 +28,7 @@ if TYPE_CHECKING: Set[achallenges.KeyAuthorizationAnnotatedChallenge] ] -class ServerManager(object): +class ServerManager: """Standalone servers manager. Manager for `ACMEServer` and `ACMETLSServer` instances. @@ -43,7 +42,7 @@ class ServerManager(object): """ def __init__(self, certs, http_01_resources): - self._instances = {} # type: Dict[int, acme_standalone.BaseDualNetworkedServers] + self._instances: Dict[int, acme_standalone.BaseDualNetworkedServers] = {} self.certs = certs self.http_01_resources = http_01_resources @@ -123,15 +122,14 @@ class Authenticator(common.Plugin): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.served = collections.defaultdict(set) # type: ServedType + self.served: ServedType = collections.defaultdict(set) # Stuff below is shared across threads (i.e. servers read # values, main thread writes). Due to the nature of CPython's # GIL, the operations are safe, c.f. # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe - self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] - self.http_01_resources = set() \ - # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] + self.certs: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] = {} + self.http_01_resources: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] = set() self.servers = ServerManager(self.certs, self.http_01_resources) @@ -183,7 +181,7 @@ class Authenticator(common.Plugin): for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, servers in six.iteritems(self.servers.running()): + for port, servers in self.servers.running().items(): if not self.served[servers]: self.servers.stop(port) diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index 88e02998d..c5b436b65 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -3,20 +3,19 @@ import argparse import collections import json import logging +from typing import DefaultDict +from typing import Dict +from typing import List +from typing import Set -import six import zope.component import zope.interface from acme import challenges -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces from certbot._internal import cli +from certbot.achallenges import KeyAuthorizationAnnotatedChallenge as AnnotatedChallenge from certbot.compat import filesystem from certbot.compat import os from certbot.display import ops @@ -68,11 +67,10 @@ to serve all files under specified web root ({0}).""" def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.full_roots = {} # type: Dict[str, str] - self.performed = collections.defaultdict(set) \ - # type: DefaultDict[str, Set[achallenges.KeyAuthorizationAnnotatedChallenge]] + self.full_roots: Dict[str, str] = {} + self.performed: DefaultDict[str, Set[AnnotatedChallenge]] = collections.defaultdict(set) # stack of dirs successfully created by this authenticator - self._created_dirs = [] # type: List[str] + self._created_dirs: List[str] = [] def prepare(self): # pylint: disable=missing-function-docstring pass @@ -92,7 +90,7 @@ to serve all files under specified web root ({0}).""" for achall in achalls: self.conf("map").setdefault(achall.domain, webroot_path) else: - known_webroots = list(set(six.itervalues(self.conf("map")))) + known_webroots = list(set(self.conf("map").values())) for achall in achalls: if achall.domain not in self.conf("map"): new_webroot = self._prompt_for_webroot(achall.domain, @@ -225,7 +223,7 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - not_removed = [] # type: List[str] + not_removed: List[str] = [] while self._created_dirs: path = self._created_dirs.pop() try: @@ -242,7 +240,7 @@ class _WebrootMapAction(argparse.Action): """Action class for parsing webroot_map.""" def __call__(self, parser, namespace, webroot_map, option_string=None): - for domains, webroot_path in six.iteritems(json.loads(webroot_map)): + for domains, webroot_path in json.loads(webroot_map).items(): webroot_path = _validate_webroot(webroot_path) namespace.webroot_map.update( (d, webroot_path) for d in cli.add_domains(namespace, domains)) diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index f4c7b4502..f29709ef4 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -1,5 +1,4 @@ """Functionality for autorenewal and associated juggling of configurations""" -from __future__ import print_function import copy import itertools @@ -8,29 +7,29 @@ import random import sys import time import traceback +from typing import List +from typing import Optional from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import ec, rsa +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.serialization import load_pem_private_key import OpenSSL -import six import zope.component -from acme.magic_typing import List -from acme.magic_typing import Optional # pylint: disable=unused-import from certbot import crypto_util -from certbot.display import util as display_util from certbot import errors from certbot import interfaces from certbot import util from certbot._internal import cli -from certbot._internal import client # pylint: disable=unused-import +from certbot._internal import client from certbot._internal import constants from certbot._internal import hooks from certbot._internal import storage from certbot._internal import updater from certbot._internal.plugins import disco as plugins_disco from certbot.compat import os +from certbot.display import util as display_util logger = logging.getLogger(__name__) @@ -120,7 +119,7 @@ def _restore_webroot_config(config, renewalparams): # see https://github.com/certbot/certbot/pull/7095 if "webroot_path" in renewalparams and not cli.set_by_cli("webroot_path"): wp = renewalparams["webroot_path"] - if isinstance(wp, six.string_types): # prior to 0.1.0, webroot_path was a string + if isinstance(wp, str): # prior to 0.1.0, webroot_path was a string wp = [wp] config.webroot_path = wp @@ -144,7 +143,7 @@ def _restore_plugin_configs(config, renewalparams): # longer defined, stored copies of that parameter will be # deserialized as strings by this logic even if they were # originally meant to be some other type. - plugin_prefixes = [] # type: List[str] + plugin_prefixes: List[str] = [] if renewalparams["authenticator"] == "webroot": _restore_webroot_config(config, renewalparams) else: @@ -155,7 +154,7 @@ def _restore_plugin_configs(config, renewalparams): for plugin_prefix in set(plugin_prefixes): plugin_prefix = plugin_prefix.replace('-', '_') - for config_item, config_value in six.iteritems(renewalparams): + for config_item, config_value in renewalparams.items(): if config_item.startswith(plugin_prefix + "_") and not cli.set_by_cli(config_item): # Values None, True, and False need to be treated specially, # As their types aren't handled correctly by configobj @@ -180,9 +179,9 @@ def restore_required_config_elements(config, renewalparams): required_items = itertools.chain( (("pref_challs", _restore_pref_challs),), - six.moves.zip(BOOL_CONFIG_ITEMS, itertools.repeat(_restore_bool)), - six.moves.zip(INT_CONFIG_ITEMS, itertools.repeat(_restore_int)), - six.moves.zip(STR_CONFIG_ITEMS, itertools.repeat(_restore_str))) + zip(BOOL_CONFIG_ITEMS, itertools.repeat(_restore_bool)), + zip(INT_CONFIG_ITEMS, itertools.repeat(_restore_int)), + zip(STR_CONFIG_ITEMS, itertools.repeat(_restore_str))) for item_name, restore_func in required_items: if item_name in renewalparams and not cli.set_by_cli(item_name): value = restore_func(item_name, renewalparams[item_name]) @@ -220,7 +219,7 @@ def _restore_pref_challs(unused_name, value): # If pref_challs has only one element, configobj saves the value # with a trailing comma so it's parsed as a list. If this comma is # removed by the user, the value is parsed as a str. - value = [value] if isinstance(value, six.string_types) else value + value = [value] if isinstance(value, str) else value return cli.parse_preferred_challenges(value) @@ -291,7 +290,7 @@ def _restore_str(name, value): def should_renew(config, lineage): - "Return true if any of the circumstances for automatic renewal apply." + """Return true if any of the circumstances for automatic renewal apply.""" if config.renew_by_default: logger.debug("Auto-renewal forced with --force-renewal...") return True @@ -306,19 +305,16 @@ def should_renew(config, lineage): def _avoid_invalidating_lineage(config, lineage, original_server): - "Do not renew a valid cert with one from a staging server!" + """Do not renew a valid cert with one from a staging server!""" # Some lineages may have begun with --staging, but then had production # certificates added to them with open(lineage.cert) as the_file: contents = the_file.read() latest_cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, contents) - # all our test certificates are from happy hacker fake CA, though maybe one day - # we should test more methodically - now_valid = "fake" not in repr(latest_cert.get_issuer()).lower() if util.is_staging(config.server): - if not util.is_staging(original_server) or now_valid: + if not util.is_staging(original_server): if not config.break_my_certs: names = ", ".join(lineage.names()) raise errors.Error( @@ -327,8 +323,8 @@ def _avoid_invalidating_lineage(config, lineage, original_server): "unless you use the --break-my-certs flag!".format(names)) -def renew_cert(config, domains, le_client, lineage): - # type: (interfaces.IConfig, Optional[List[str]], client.Client, storage.RenewableCert) -> None +def renew_cert(config: interfaces.IConfig, domains: Optional[List[str]], le_client: client.Client, + lineage: storage.RenewableCert) -> None: """Renew a certificate lineage.""" renewal_params = lineage.configuration["renewalparams"] original_server = renewal_params.get("server", cli.flag_default("server")) @@ -355,14 +351,14 @@ def renew_cert(config, domains, le_client, lineage): def report(msgs, category): - "Format a results report for a category of renewal outcomes" + """Format a results report for a category of renewal outcomes""" lines = ("%s (%s)" % (m, category) for m in msgs) return " " + "\n ".join(lines) -def _renew_describe_results(config, renew_successes, renew_failures, - renew_skipped, parse_failures): - # type: (interfaces.IConfig, List[str], List[str], List[str], List[str]) -> None +def _renew_describe_results(config: interfaces.IConfig, renew_successes: List[str], + renew_failures: List[str], renew_skipped: List[str], + parse_failures: List[str]) -> None: """ Print a report to the terminal about the results of the renewal process. @@ -515,8 +511,7 @@ def handle_renewal_request(config): logger.debug("no renewal failures") -def _update_renewal_params_from_key(key_path, config): - # type: (str, interfaces.IConfig) -> None +def _update_renewal_params_from_key(key_path: str, config: interfaces.IConfig) -> None: with open(key_path, 'rb') as file_h: key = load_pem_private_key(file_h.read(), password=None, backend=default_backend()) if isinstance(key, rsa.RSAPrivateKey): diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index d781f3d9d..0268fbf34 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -1,12 +1,10 @@ """Collects and displays information to the user.""" -from __future__ import print_function - import collections import logging +import queue import sys import textwrap -from six.moves import queue # type: ignore import zope.interface from certbot import interfaces @@ -16,7 +14,7 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IReporter) -class Reporter(object): +class Reporter: """Collects and displays information to the user. :ivar `queue.PriorityQueue` messages: Messages to be displayed to @@ -34,7 +32,7 @@ class Reporter(object): _msg_type = collections.namedtuple('ReporterMsg', 'priority text on_crash') def __init__(self, config): - self.messages = queue.PriorityQueue() # type: queue.PriorityQueue[Reporter._msg_type] + self.messages: queue.PriorityQueue[Reporter._msg_type] = queue.PriorityQueue() self.config = config def add_message(self, msg, priority, on_crash=True): diff --git a/certbot/certbot/_internal/snap_config.py b/certbot/certbot/_internal/snap_config.py index a90740787..1ba53cad4 100644 --- a/certbot/certbot/_internal/snap_config.py +++ b/certbot/certbot/_internal/snap_config.py @@ -1,13 +1,13 @@ """Module configuring Certbot in a snap environment""" import logging import socket +from typing import List from requests import Session from requests.adapters import HTTPAdapter from requests.exceptions import HTTPError from requests.exceptions import RequestException -from acme.magic_typing import List from certbot.compat import os from certbot.errors import Error @@ -33,8 +33,7 @@ _ARCH_TRIPLET_MAP = { LOGGER = logging.getLogger(__name__) -def prepare_env(cli_args): - # type: (List[str]) -> List[str] +def prepare_env(cli_args: List[str]) -> List[str]: """ Prepare runtime environment for a certbot execution in snap. :param list cli_args: List of command line arguments diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index ff58313e5..218a5dd92 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -6,10 +6,10 @@ import re import shutil import stat +from typing import Optional import configobj import parsedatetime import pytz -import six from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.hazmat.primitives.serialization import load_pem_private_key @@ -59,7 +59,7 @@ def renewal_file_for_certname(config, certname): return path -def cert_path_for_cert_name(config, cert_name): +def cert_path_for_cert_name(config: interfaces.IConfig, cert_name: str) -> str: """ If `--cert-name` was specified, but you need a value for `--cert-path`. :param `configuration.NamespaceConfig` config: parsed command line arguments @@ -67,10 +67,7 @@ def cert_path_for_cert_name(config, cert_name): """ cert_name_implied_conf = renewal_file_for_certname(config, cert_name) - fullchain_path = configobj.ConfigObj(cert_name_implied_conf)["fullchain"] - with open(fullchain_path) as f: - cert_path = (fullchain_path, f.read()) - return cert_path + return configobj.ConfigObj(cert_name_implied_conf)["fullchain"] def config_with_defaults(config=None): @@ -275,7 +272,7 @@ def relevant_values(all_values): rv = dict( (option, value) - for option, value in six.iteritems(all_values) + for option, value in all_values.items() if _relevant(namespaces, option) and cli.option_was_set(option, value)) # We always save the server value to help with forward compatibility # and behavioral consistency when versions of Certbot with different @@ -522,11 +519,15 @@ class RenewableCert(interfaces.RenewableCert): return _relpath_from_file(self.archive_dir, from_file) @property - def is_test_cert(self): + def server(self) -> Optional[str]: + """Returns the ACME server associated with this certificate""" + return self.configuration["renewalparams"].get("server", None) + + @property + def is_test_cert(self) -> bool: """Returns true if this is a test cert from a staging server.""" - server = self.configuration["renewalparams"].get("server", None) - if server: - return util.is_staging(server) + if self.server: + return util.is_staging(self.server) return False def _check_symlinks(self): diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index 0152685e9..25e1687da 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -5,8 +5,7 @@ import errno import os # pylint: disable=os-module-forbidden import stat import sys - -from acme.magic_typing import List +from typing import List try: import ntsecuritycon @@ -36,8 +35,7 @@ class _WindowsUmask: _WINDOWS_UMASK = _WindowsUmask() -def chmod(file_path, mode): - # type: (str, int) -> None +def chmod(file_path: str, mode: int) -> None: """ Apply a POSIX mode on given file_path: @@ -58,8 +56,7 @@ def chmod(file_path, mode): _apply_win_mode(file_path, mode) -def umask(mask): - # type: (int) -> int +def umask(mask: int) -> int: """ Set the current numeric umask and return the previous umask. On Linux, the built-in umask method is used. On Windows, our Certbot-side implementation is used. @@ -85,8 +82,8 @@ def umask(mask): # Since copying and editing arbitrary DACL is very difficult, and since we actually know # the mode to apply at the time the owner of a file should change, it is easier to just # change the owner, then reapply the known mode, as copy_ownership_and_apply_mode() does. -def copy_ownership_and_apply_mode(src, dst, mode, copy_user, copy_group): - # type: (str, str, int, bool, bool) -> None +def copy_ownership_and_apply_mode(src: str, dst: str, mode: int, + copy_user: bool, copy_group: bool) -> None: """ Copy ownership (user and optionally group on Linux) from the source to the destination, then apply given mode in compatible way for Linux and Windows. @@ -118,8 +115,8 @@ def copy_ownership_and_apply_mode(src, dst, mode, copy_user, copy_group): # equivalent POSIX mode, because ownership and mode are copied altogether on the destination # file, so no recomputing of the DACL against the new owner is needed, as it would be # for a copy_ownership alone method. -def copy_ownership_and_mode(src, dst, copy_user=True, copy_group=True): - # type: (str, str, bool, bool) -> None +def copy_ownership_and_mode(src: str, dst: str, + copy_user: bool = True, copy_group: bool = True) -> None: """ Copy ownership (user and optionally group on Linux) and mode/DACL from the source to the destination. @@ -143,8 +140,7 @@ def copy_ownership_and_mode(src, dst, copy_user=True, copy_group=True): _copy_win_mode(src, dst) -def check_mode(file_path, mode): - # type: (str, int) -> bool +def check_mode(file_path: str, mode: int) -> bool: """ Check if the given mode matches the permissions of the given file. On Linux, will make a direct comparison, on Windows, mode will be compared against @@ -161,8 +157,7 @@ def check_mode(file_path, mode): return _check_win_mode(file_path, mode) -def check_owner(file_path): - # type: (str) -> bool +def check_owner(file_path: str) -> bool: """ Check if given file is owned by current user. @@ -184,8 +179,7 @@ def check_owner(file_path): return _get_current_user() == user -def check_permissions(file_path, mode): - # type: (str, int) -> bool +def check_permissions(file_path: str, mode: int) -> bool: """ Check if given file has the given mode and is owned by current user. @@ -197,8 +191,7 @@ def check_permissions(file_path, mode): return check_owner(file_path) and check_mode(file_path, mode) -def open(file_path, flags, mode=0o777): # pylint: disable=redefined-builtin - # type: (str, int, int) -> int +def open(file_path: str, flags: int, mode: int = 0o777) -> int: # pylint: disable=redefined-builtin """ Wrapper of original os.open function, that will ensure on Windows that given mode is correctly applied. @@ -267,8 +260,7 @@ def open(file_path, flags, mode=0o777): # pylint: disable=redefined-builtin return handle -def makedirs(file_path, mode=0o777): - # type: (str, int) -> None +def makedirs(file_path: str, mode: int = 0o777) -> None: """ Rewrite of original os.makedirs function, that will ensure on Windows that given mode is correctly applied. @@ -300,8 +292,7 @@ def makedirs(file_path, mode=0o777): umask(current_umask) -def mkdir(file_path, mode=0o777): - # type: (str, int) -> None +def mkdir(file_path: str, mode: int = 0o777) -> None: """ Rewrite of original os.mkdir function, that will ensure on Windows that given mode is correctly applied. @@ -332,8 +323,7 @@ def mkdir(file_path, mode=0o777): return None -def replace(src, dst): - # type: (str, str) -> None +def replace(src: str, dst: str) -> None: """ Rename a file to a destination path and handles situations where the destination exists. @@ -350,8 +340,7 @@ def replace(src, dst): os.rename(src, dst) -def realpath(file_path): - # type: (str) -> str +def realpath(file_path: str) -> str: """ Find the real path for the given path. This method resolves symlinks, including recursive symlinks, and is protected against symlinks that creates an infinite loop. @@ -372,7 +361,7 @@ def realpath(file_path): raise RuntimeError('Error, link {0} is a loop!'.format(original_path)) return path - inspected_paths = [] # type: List[str] + inspected_paths: List[str] = [] while os.path.islink(file_path): link_path = file_path file_path = os.readlink(file_path) @@ -385,8 +374,7 @@ def realpath(file_path): return os.path.abspath(file_path) -def readlink(link_path): - # type: (str) -> str +def readlink(link_path: str) -> str: """ Return a string representing the path to which the symbolic link points. @@ -420,8 +408,7 @@ def readlink(link_path): # elevated privileges or not. However this is not a problem since certbot always # requires to be run under a privileged shell, so the user will always benefit # from the highest (privileged one) set of permissions on a given file. -def is_executable(path): - # type: (str) -> bool +def is_executable(path: str) -> bool: """ Is path an executable file? @@ -435,8 +422,7 @@ def is_executable(path): return _win_is_executable(path) -def has_world_permissions(path): - # type: (str) -> bool +def has_world_permissions(path: str) -> bool: """ Check if everybody/world has any right (read/write/execute) on a file given its path. @@ -457,8 +443,7 @@ def has_world_permissions(path): })) -def compute_private_key_mode(old_key, base_mode): - # type: (str, int) -> int +def compute_private_key_mode(old_key: str, base_mode: int) -> int: """ Calculate the POSIX mode to apply to a private key given the previous private key. @@ -479,8 +464,7 @@ def compute_private_key_mode(old_key, base_mode): return base_mode -def has_same_ownership(path1, path2): - # type: (str, str) -> bool +def has_same_ownership(path1: str, path2: str) -> bool: """ Return True if the ownership of two files given their respective path is the same. On Windows, ownership is checked against owner only, since files do not have a group owner. @@ -505,8 +489,7 @@ def has_same_ownership(path1, path2): return user1 == user2 -def has_min_permissions(path, min_mode): - # type: (str, int) -> bool +def has_min_permissions(path: str, min_mode: int) -> bool: """ Check if a file given its path has at least the permissions defined by the given minimal mode. On Windows, group permissions are ignored since files do not have a group owner. diff --git a/certbot/certbot/compat/misc.py b/certbot/certbot/compat/misc.py index f4ea4a5cc..3d2624906 100644 --- a/certbot/certbot/compat/misc.py +++ b/certbot/certbot/compat/misc.py @@ -8,12 +8,12 @@ import logging import select import subprocess import sys +from typing import Optional +from typing import Tuple from certbot import errors from certbot.compat import os -from acme.magic_typing import Tuple, Optional - try: from win32com.shell import shell as shellwin32 POSIX_MODE = False @@ -27,8 +27,7 @@ logger = logging.getLogger(__name__) STANDARD_BINARY_DIRS = ["/usr/sbin", "/usr/local/bin", "/usr/local/sbin"] if POSIX_MODE else [] -def raise_for_non_administrative_windows_rights(): - # type: () -> None +def raise_for_non_administrative_windows_rights() -> None: """ On Windows, raise if current shell does not have the administrative rights. Do nothing on Linux. @@ -39,8 +38,7 @@ def raise_for_non_administrative_windows_rights(): raise errors.Error('Error, certbot must be run on a shell with administrative rights.') -def readline_with_timeout(timeout, prompt): - # type: (float, str) -> str +def readline_with_timeout(timeout: float, prompt: str) -> str: """ Read user input to return the first line entered, or raise after specified timeout. @@ -81,8 +79,7 @@ LINUX_DEFAULT_FOLDERS = { } -def get_default_folder(folder_type): - # type: (str) -> str +def get_default_folder(folder_type: str) -> str: """ Return the relevant default folder for the current OS @@ -99,8 +96,7 @@ def get_default_folder(folder_type): return WINDOWS_DEFAULT_FOLDERS[folder_type] -def underscores_for_unsupported_characters_in_path(path): - # type: (str) -> str +def underscores_for_unsupported_characters_in_path(path: str) -> str: """ Replace unsupported characters in path for current OS by underscores. :param str path: the path to normalize @@ -116,8 +112,7 @@ def underscores_for_unsupported_characters_in_path(path): return drive + tail.replace(':', '_') -def execute_command(cmd_name, shell_cmd, env=None): - # type: (str, str, Optional[dict]) -> Tuple[str, str] +def execute_command(cmd_name: str, shell_cmd: str, env: Optional[dict] = None) -> Tuple[str, str]: """ Run a command: - on Linux command will be run by the standard shell selected with Popen(shell=True) diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 6feb6e4b9..92ecd1679 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -6,27 +6,29 @@ """ import hashlib import logging +import re import warnings -import re +from typing import List # See https://github.com/pyca/cryptography/issues/4275 from cryptography import x509 # type: ignore -from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm +from cryptography.exceptions import InvalidSignature +from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import ec -from cryptography.hazmat.primitives.asymmetric.ec import ECDSA, EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey -from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat +from cryptography.hazmat.primitives.serialization import Encoding +from cryptography.hazmat.primitives.serialization import NoEncryption +from cryptography.hazmat.primitives.serialization import PrivateFormat from OpenSSL import crypto from OpenSSL import SSL # type: ignore - import pyrfc3339 -import six import zope.component from acme import crypto_util as acme_crypto_util -from acme.magic_typing import IO, Any, List # pylint: disable=unused-import from certbot import errors from certbot import interfaces from certbot import util @@ -215,7 +217,7 @@ def make_key(bits=1024, key_type="rsa", elliptic_curve=None): except TypeError: raise errors.Error("Unsupported elliptic curve: {}".format(elliptic_curve)) except UnsupportedAlgorithm as e: - raise six.raise_from(e, errors.Error(str(e))) + raise e from errors.Error(str(e)) _key_pem = _key.private_bytes( encoding=Encoding.PEM, format=PrivateFormat.TraditionalOpenSSL, @@ -270,9 +272,9 @@ def verify_renewable_cert_sig(renewable_cert): :raises errors.Error: If signature verification fails. """ try: - with open(renewable_cert.chain_path, 'rb') as chain_file: # type: IO[bytes] + with open(renewable_cert.chain_path, 'rb') as chain_file: chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) - with open(renewable_cert.cert_path, 'rb') as cert_file: # type: IO[bytes] + with open(renewable_cert.cert_path, 'rb') as cert_file: cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) pk = chain.public_key() with warnings.catch_warnings(): @@ -347,11 +349,11 @@ def verify_fullchain(renewable_cert): :raises errors.Error: If cert and chain do not combine to fullchain. """ try: - with open(renewable_cert.chain_path) as chain_file: # type: IO[str] + with open(renewable_cert.chain_path) as chain_file: chain = chain_file.read() - with open(renewable_cert.cert_path) as cert_file: # type: IO[str] + with open(renewable_cert.cert_path) as cert_file: cert = cert_file.read() - with open(renewable_cert.fullchain_path) as fullchain_file: # type: IO[str] + with open(renewable_cert.fullchain_path) as fullchain_file: fullchain = fullchain_file.read() if (cert + chain) != fullchain: error_str = "fullchain does not match cert + chain for {0}!" @@ -437,8 +439,7 @@ def get_names_from_cert(csr, typ=crypto.FILETYPE_PEM): csr, crypto.load_certificate, typ) -def get_names_from_req(csr, typ=crypto.FILETYPE_PEM): - # type: (str, Any) -> List[str] +def get_names_from_req(csr: str, typ: int = crypto.FILETYPE_PEM) -> List[str]: """Get a list of domains from a CSR, including the CN if it is set. :param str cert: CSR (encoded). @@ -498,21 +499,16 @@ def _notAfterBefore(cert_path, method): """ # pylint: disable=redefined-outer-name - with open(cert_path, "rb") as f: # type: IO[bytes] + with open(cert_path, "rb") as f: x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) # pyopenssl always returns bytes timestamp = method(x509) reformatted_timestamp = [timestamp[0:4], b"-", timestamp[4:6], b"-", timestamp[6:8], b"T", timestamp[8:10], b":", timestamp[10:12], b":", timestamp[12:]] - # pyrfc3339 always uses the type `str`. This means that in Python 2, it - # expects str/bytes and in Python 3 it expects its str type or the Python 2 - # equivalent of the type unicode. + # pyrfc3339 always uses the type `str` timestamp_bytes = b"".join(reformatted_timestamp) - if six.PY3: - timestamp_str = timestamp_bytes.decode('ascii') - else: - timestamp_str = timestamp_bytes + timestamp_str = timestamp_bytes.decode('ascii') return pyrfc3339.parse(timestamp_str) @@ -580,7 +576,7 @@ def get_serial_from_cert(cert_path): :rtype: int """ # pylint: disable=redefined-outer-name - with open(cert_path, "rb") as f: # type: IO[bytes] + with open(cert_path, "rb") as f: x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) return x509.get_serial_number() diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index c48454637..9443ae266 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -12,11 +12,11 @@ Other messages can use the `logging` module. See `log.py`. import logging import sys import textwrap +from typing import List -import zope.interface import zope.component +import zope.interface -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot._internal import constants @@ -71,7 +71,7 @@ def _wrap_lines(msg): def input_with_timeout(prompt=None, timeout=36000.0): """Get user input with a timeout. - Behaves the same as six.moves.input, however, an error is raised if + Behaves the same as the builtin input, however, an error is raised if a user doesn't answer after timeout seconds. The default timeout value was chosen to place it just under 12 hours for users following our advice and running Certbot twice a day. @@ -85,7 +85,7 @@ def input_with_timeout(prompt=None, timeout=36000.0): :raises errors.Error if no answer is given before the timeout """ - # use of sys.stdin and sys.stdout to mimic six.moves.input based on + # use of sys.stdin and sys.stdout to mimic the builtin input based on # https://github.com/python/cpython/blob/baf7bb30a02aabde260143136bdf5b3738a1d409/Lib/getpass.py#L129 if prompt: sys.stdout.write(prompt) @@ -98,8 +98,7 @@ def input_with_timeout(prompt=None, timeout=36000.0): return line.rstrip('\n') -def notify(msg): - # type: (str) -> None +def notify(msg: str) -> None: """Display a basic status message. :param str msg: message to display @@ -111,7 +110,7 @@ def notify(msg): @zope.interface.implementer(interfaces.IDisplay) -class FileDisplay(object): +class FileDisplay: """File-based display.""" # see https://github.com/certbot/certbot/issues/3915 @@ -478,7 +477,7 @@ def assert_valid_call(prompt, default, cli_flag, force_interactive): @zope.interface.implementer(interfaces.IDisplay) -class NoninteractiveDisplay(object): +class NoninteractiveDisplay: """An iDisplay implementation that never asks for interactive user input""" def __init__(self, outfile, *unused_args, **unused_kwargs): @@ -636,8 +635,7 @@ def _parens_around_char(label): return "({first}){rest}".format(first=label[0], rest=label[1:]) -def summarize_domain_list(domains): - # type: (List[str]) -> str +def summarize_domain_list(domains: List[str]) -> str: """Summarizes a list of domains in the format of: example.com.com and N more domains or if there is are only two domains: diff --git a/certbot/certbot/interfaces.py b/certbot/certbot/interfaces.py index 28c6f2ac1..ddbee8ddc 100644 --- a/certbot/certbot/interfaces.py +++ b/certbot/certbot/interfaces.py @@ -1,14 +1,12 @@ """Certbot client interfaces.""" import abc -import six import zope.interface # pylint: disable=no-self-argument,no-method-argument,inherit-non-class -@six.add_metaclass(abc.ABCMeta) -class AccountStorage(object): +class AccountStorage(object, metaclass=abc.ABCMeta): """Accounts storage interface.""" @abc.abstractmethod @@ -547,8 +545,7 @@ class IReporter(zope.interface.Interface): """Prints messages to the user and clears the message queue.""" -@six.add_metaclass(abc.ABCMeta) -class RenewableCert(object): +class RenewableCert(object, metaclass=abc.ABCMeta): """Interface to a certificate lineage.""" @abc.abstractproperty @@ -613,8 +610,7 @@ class RenewableCert(object): # an update during the run or install subcommand, it should do so when # :func:`IInstaller.deploy_cert` is called. -@six.add_metaclass(abc.ABCMeta) -class GenericUpdater(object): +class GenericUpdater(object, metaclass=abc.ABCMeta): """Interface for update types not currently specified by Certbot. This class allows plugins to perform types of updates that Certbot hasn't @@ -646,8 +642,7 @@ class GenericUpdater(object): """ -@six.add_metaclass(abc.ABCMeta) -class RenewDeployer(object): +class RenewDeployer(object, metaclass=abc.ABCMeta): """Interface for update types run when a lineage is renewed This class allows plugins to perform types of updates that need to run at diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index b63338e2e..0a842e108 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -5,6 +5,8 @@ import logging import re from subprocess import PIPE from subprocess import Popen +from typing import Optional +from typing import Tuple from cryptography import x509 from cryptography.exceptions import InvalidSignature @@ -16,8 +18,6 @@ from cryptography.hazmat.primitives import serialization import pytz import requests -from acme.magic_typing import Optional -from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import util @@ -36,7 +36,7 @@ except (ImportError, AttributeError): # pragma: no cover logger = logging.getLogger(__name__) -class RevocationChecker(object): +class RevocationChecker: """This class figures out OCSP checking on this system, and performs it.""" def __init__(self, enforce_openssl_binary_usage=False): @@ -59,8 +59,7 @@ class RevocationChecker(object): else: self.host_args = lambda host: ["Host", host] - def ocsp_revoked(self, cert): - # type: (RenewableCert) -> bool + def ocsp_revoked(self, cert: RenewableCert) -> bool: """Get revoked status for a particular cert version. .. todo:: Make this a non-blocking call @@ -72,8 +71,7 @@ class RevocationChecker(object): """ return self.ocsp_revoked_by_paths(cert.cert_path, cert.chain_path) - def ocsp_revoked_by_paths(self, cert_path, chain_path, timeout=10): - # type: (str, str, int) -> bool + def ocsp_revoked_by_paths(self, cert_path: str, chain_path: str, timeout: int = 10) -> bool: """Performs the OCSP revocation check :param str cert_path: Certificate filepath @@ -102,8 +100,8 @@ class RevocationChecker(object): return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url, timeout) return _check_ocsp_cryptography(cert_path, chain_path, url, timeout) - def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url, timeout): - # type: (str, str, str, str, int) -> bool + def _check_ocsp_openssl_bin(self, cert_path: str, chain_path: str, + host: str, url: str, timeout: int) -> bool: # Minimal implementation of proxy selection logic as seen in, e.g., cURL # Some things that won't work, but may well be in use somewhere: # - username and password for proxy authentication @@ -140,8 +138,7 @@ class RevocationChecker(object): return _translate_ocsp_query(cert_path, output, err) -def _determine_ocsp_server(cert_path): - # type: (str) -> Tuple[Optional[str], Optional[str]] +def _determine_ocsp_server(cert_path: str) -> Tuple[Optional[str], Optional[str]]: """Extract the OCSP server host from a certificate. :param str cert_path: Path to the cert we're checking OCSP for @@ -171,8 +168,7 @@ def _determine_ocsp_server(cert_path): return None, None -def _check_ocsp_cryptography(cert_path, chain_path, url, timeout): - # type: (str, str, str, int) -> bool +def _check_ocsp_cryptography(cert_path: str, chain_path: str, url: str, timeout: int) -> bool: # Retrieve OCSP response with open(chain_path, 'rb') as file_handler: issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index d3fb5b7dc..06acefc69 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -3,13 +3,13 @@ import logging import re import shutil import tempfile +from typing import List from josepy import util as jose_util import pkg_resources import zope.interface -from acme.magic_typing import List -from certbot import achallenges # pylint: disable=unused-import +from certbot import achallenges from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -40,7 +40,7 @@ hostname_regex = re.compile( @zope.interface.implementer(interfaces.IPlugin) -class Plugin(object): +class Plugin: """Generic plugin.""" # provider is not inherited, subclasses must define it on their own # @zope.interface.provider(interfaces.IPluginFactory) @@ -201,7 +201,7 @@ class Installer(Plugin): constants.ALL_SSL_DHPARAMS_HASHES) -class Addr(object): +class Addr: r"""Represents an virtual host address. :param str addr: addr part of vhost address @@ -299,7 +299,7 @@ class Addr(object): return result -class ChallengePerformer(object): +class ChallengePerformer: """Abstract base for challenge performers. :ivar configurator: Authenticator and installer plugin @@ -313,8 +313,8 @@ class ChallengePerformer(object): def __init__(self, configurator): self.configurator = configurator - self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] - self.indices = [] # type: List[int] + self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = [] + self.indices: List[int] = [] def add_chall(self, achall, idx=None): """Store challenge to be performed when perform() is called. diff --git a/certbot/certbot/plugins/dns_common.py b/certbot/certbot/plugins/dns_common.py index 245b7dc05..5c0fbcba9 100644 --- a/certbot/certbot/plugins/dns_common.py +++ b/certbot/certbot/plugins/dns_common.py @@ -233,7 +233,7 @@ class DNSAuthenticator(common.Plugin): raise errors.PluginError('{0} required to proceed.'.format(label)) -class CredentialsConfiguration(object): +class CredentialsConfiguration: """Represents a user-supplied filed which stores API credentials.""" def __init__(self, filename, mapper=lambda x: x): diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index c3d80ca29..10efa5f85 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -1,12 +1,12 @@ """Common code for DNS Authenticator Plugins built on Lexicon.""" import logging +from typing import Any +from typing import Dict +from typing import Union from requests.exceptions import HTTPError from requests.exceptions import RequestException -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import Union from certbot import errors from certbot.plugins import dns_common @@ -23,7 +23,7 @@ except ImportError: logger = logging.getLogger(__name__) -class LexiconClient(object): +class LexiconClient: """ Encapsulates all communication with a DNS provider via Lexicon. """ @@ -116,8 +116,9 @@ class LexiconClient(object): return None -def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_options): - # type: (str, Dict, Dict) -> Union[ConfigResolver, Dict] +def build_lexicon_config(lexicon_provider_name: str, + lexicon_options: Dict, provider_options: Dict + ) -> Union[ConfigResolver, Dict]: """ Convenient function to build a Lexicon 2.x/3.x config object. :param str lexicon_provider_name: the name of the lexicon provider to use @@ -126,7 +127,7 @@ def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_option :return: configuration to apply to the provider :rtype: ConfigurationResolver or dict """ - config = {'provider_name': lexicon_provider_name} # type: Dict[str, Any] + config: Dict[str, Any] = {'provider_name': lexicon_provider_name} config.update(lexicon_options) if not ConfigResolver: # Lexicon 2.x diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index d5044d336..1affcae4e 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -6,7 +6,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six from acme import challenges from certbot import achallenges @@ -18,7 +17,7 @@ DOMAIN = 'example.com' KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) -class BaseAuthenticatorTest(object): +class BaseAuthenticatorTest: """ A base test class to reduce duplication between test code for DNS Authenticator Plugins. @@ -31,7 +30,7 @@ class BaseAuthenticatorTest(object): challb=acme_util.DNS01, domain=DOMAIN, account_key=KEY) def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) # pylint: disable=no-member + self.assertTrue(isinstance(self.auth.more_info(), str)) # pylint: disable=no-member def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref(None), [challenges.DNS01]) # pylint: disable=no-member diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index 1bef06042..adeb1a268 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -34,7 +34,7 @@ class BaseLexiconAuthenticatorTest(dns_test_common.BaseAuthenticatorTest): self.assertEqual(expected, self.mock_client.mock_calls) -class BaseLexiconClientTest(object): +class BaseLexiconClientTest: DOMAIN_NOT_FOUND = Exception('No domain found') GENERIC_ERROR = RequestException LOGIN_ERROR = HTTPError('400 Client Error: ...') diff --git a/certbot/certbot/plugins/enhancements.py b/certbot/certbot/plugins/enhancements.py index 4abce2d2f..c1de4d44f 100644 --- a/certbot/certbot/plugins/enhancements.py +++ b/certbot/certbot/plugins/enhancements.py @@ -1,11 +1,9 @@ """New interface style Certbot enhancements""" import abc +from typing import Any +from typing import Dict +from typing import List -import six - -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot._internal import constants ENHANCEMENTS = ["redirect", "ensure-http-header", "ocsp-stapling"] @@ -91,8 +89,7 @@ def populate_cli(add): help=enh["cli_help"]) -@six.add_metaclass(abc.ABCMeta) -class AutoHSTSEnhancement(object): +class AutoHSTSEnhancement(object, metaclass=abc.ABCMeta): """ Enhancement interface that installer plugins can implement in order to provide functionality that configures the software to have a @@ -159,7 +156,7 @@ class AutoHSTSEnhancement(object): # This is used to configure internal new style enhancements in Certbot. These # enhancement interfaces need to be defined in this file. Please do not modify # this list from plugin code. -_INDEX = [ +_INDEX: List[Dict[str, Any]] = [ { "name": "AutoHSTS", "cli_help": "Gradually increasing max-age value for HTTP Strict Transport "+ @@ -174,4 +171,4 @@ _INDEX = [ "deployer_function": "deploy_autohsts", "enable_function": "enable_autohsts" } -] # type: List[Dict[str, Any]] +] diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index f3ed14dce..0d32d6850 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -1,9 +1,9 @@ """Plugin storage class.""" import json import logging +from typing import Any +from typing import Dict -from acme.magic_typing import Any -from acme.magic_typing import Dict from certbot import errors from certbot.compat import filesystem from certbot.compat import os @@ -11,7 +11,7 @@ from certbot.compat import os logger = logging.getLogger(__name__) -class PluginStorage(object): +class PluginStorage: """Class implementing storage functionality for plugins""" def __init__(self, config, classkey): @@ -42,7 +42,7 @@ class PluginStorage(object): :raises .errors.PluginStorageError: when unable to open or read the file """ - data = {} # type: Dict[str, Any] + data: Dict[str, Any] = {} filedata = "" try: with open(self._storagepath, 'r') as fh: diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index 58e1216b7..e6777a7da 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -3,11 +3,9 @@ import csv import glob import logging import shutil -import sys import time import traceback -import six from certbot import errors from certbot import util @@ -18,7 +16,7 @@ from certbot.compat import os logger = logging.getLogger(__name__) -class Reverter(object): +class Reverter: """Reverter Class - save and revert configuration checkpoints. This class can be used by the plugins, especially Installers, to @@ -252,11 +250,10 @@ class Reverter(object): def _run_undo_commands(self, filepath): """Run all commands in a file.""" - # NOTE: csv module uses native strings. That is, bytes on Python 2 and - # unicode on Python 3 + # NOTE: csv module uses native strings. That is unicode on Python 3 # It is strongly advised to set newline = '' on Python 3 with CSV, # and it fixes problems on Windows. - kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + kwargs = {'newline': ''} with open(filepath, 'r', **kwargs) as csvfile: # type: ignore csvreader = csv.reader(csvfile) for command in reversed(list(csvreader)): @@ -355,7 +352,7 @@ class Reverter(object): command_file = None # It is strongly advised to set newline = '' on Python 3 with CSV, # and it fixes problems on Windows. - kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + kwargs = {'newline': ''} try: if os.path.isfile(commands_fp): command_file = open(commands_fp, "a", **kwargs) # type: ignore @@ -518,7 +515,7 @@ class Reverter(object): # It is possible save checkpoints faster than 1 per second resulting in # collisions in the naming convention. - for _ in six.moves.range(2): + for _ in range(2): timestamp = self._checkpoint_timestamp() final_dir = os.path.join(self.config.backup_dir, timestamp) try: diff --git a/certbot/certbot/tests/acme_util.py b/certbot/certbot/tests/acme_util.py index f4a20ea86..49ca88bf5 100644 --- a/certbot/certbot/tests/acme_util.py +++ b/certbot/certbot/tests/acme_util.py @@ -2,7 +2,6 @@ import datetime import josepy as jose -import six from acme import challenges from acme import messages @@ -69,7 +68,7 @@ def gen_authzr(authz_status, domain, challs, statuses, combos=True): """ challbs = tuple( chall_to_challb(chall, status) - for chall, status in six.moves.zip(challs, statuses) + for chall, status in zip(challs, statuses) ) authz_kwargs = { "identifier": messages.Identifier( diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index acb31819f..78236fa06 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -1,4 +1,6 @@ """Test utilities.""" +from importlib import reload as reload_module +import io import logging from multiprocessing import Event from multiprocessing import Process @@ -11,20 +13,8 @@ import warnings from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -try: - import mock - warnings.warn( - "The external mock module is being used for backwards compatibility " - "since it is available, however, future versions of Certbot's tests will " - "use unittest.mock. Be sure to update your code accordingly.", - PendingDeprecationWarning - ) -except ImportError: # pragma: no cover - from unittest import mock # type: ignore import OpenSSL import pkg_resources -import six -from six.moves import reload_module from certbot import interfaces from certbot import util @@ -36,6 +26,18 @@ from certbot.compat import filesystem from certbot.compat import os from certbot.display import util as display_util +try: + import mock + warnings.warn( + "The external mock module is being used for backwards compatibility " + "since it is available, however, future versions of Certbot's tests will " + "use unittest.mock. Be sure to update your code accordingly.", + PendingDeprecationWarning + ) +except ImportError: # pragma: no cover + from unittest import mock # type: ignore + + def vector_path(*names): """Path to a test vector.""" @@ -177,13 +179,13 @@ def patch_get_utility_with_stdout(target='zope.component.getUtility', :rtype: mock.MagicMock """ - stdout = stdout if stdout else six.StringIO() + stdout = stdout if stdout else io.StringIO() freezable_mock = _create_get_utility_mock_with_stdout(stdout) return mock.patch(target, new=freezable_mock) -class FreezableMock(object): +class FreezableMock: """Mock object with the ability to freeze attributes. This class works like a regular mock.MagicMock object, except diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index 8db5ab34a..5f4a08dc7 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -4,7 +4,6 @@ import argparse import atexit import collections -from collections import OrderedDict import distutils.version import errno import logging @@ -13,13 +12,13 @@ import re import socket import subprocess import sys +from typing import Dict +from typing import Text +from typing import Tuple +from typing import Union import configargparse -import six -from acme.magic_typing import Text -from acme.magic_typing import Tuple -from acme.magic_typing import Union from certbot import errors from certbot._internal import constants from certbot._internal import lock @@ -59,7 +58,7 @@ _INITIAL_PID = os.getpid() # the dict are attempted to be cleaned up at program exit. If the # program exits before the lock is cleaned up, it is automatically # released, but the file isn't deleted. -_LOCKS = OrderedDict() # type: OrderedDict[str, lock.LockFile] +_LOCKS: Dict[str, lock.LockFile] = {} def env_no_snap_for_external_calls(): @@ -153,7 +152,7 @@ def lock_dir_until_exit(dir_path): def _release_locks(): - for dir_lock in six.itervalues(_LOCKS): + for dir_lock in _LOCKS.values(): try: dir_lock.release() except: # pylint: disable=bare-except @@ -217,10 +216,10 @@ def safe_open(path, mode="w", chmod=None): if ``None``. """ - open_args = () # type: Union[Tuple[()], Tuple[int]] + open_args: Union[Tuple[()], Tuple[int]] = () if chmod is not None: open_args = (chmod,) - fdopen_args = () # type: Union[Tuple[()], Tuple[int]] + fdopen_args: Union[Tuple[()], Tuple[int]] = () fd = filesystem.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args) return os.fdopen(fd, mode, *fdopen_args) @@ -517,7 +516,7 @@ def enforce_domain_sanity(domain): """ # Unicode try: - if isinstance(domain, six.binary_type): + if isinstance(domain, bytes): domain = domain.decode('utf-8') domain.encode('ascii') except UnicodeError: @@ -578,8 +577,8 @@ def is_wildcard_domain(domain): :rtype: bool """ - wildcard_marker = b"*." # type: Union[Text, bytes] - if isinstance(domain, six.text_type): + wildcard_marker: Union[Text, bytes] = b"*." + if isinstance(domain, str): wildcard_marker = u"*." return domain.startswith(wildcard_marker) diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index 4482ea439..27511777b 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -118,7 +118,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.12.0 + "". (default: CertbotACMEClient/1.13.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -167,19 +167,6 @@ automation: --duplicate Allow making a certificate lineage that duplicates an existing one (both can be renewed in parallel) (default: False) - --os-packages-only (certbot-auto only) install OS package dependencies - and then stop (default: False) - --no-self-upgrade (certbot-auto only) prevent the certbot-auto script - from upgrading itself to newer released versions - (default: Upgrade automatically) - --no-bootstrap (certbot-auto only) prevent the certbot-auto script - from installing OS-level dependencies (default: Prompt - to install OS-wide dependencies, but exit if the user - says 'No') - --no-permissions-check - (certbot-auto only) skip the check on the file system - permissions of the certbot-auto script (default: - False) -q, --quiet Silence all output except errors. Useful for automation via cron. Implies --non-interactive. (default: False) @@ -254,8 +241,8 @@ paths: Flags for changing execution paths & servers --cert-path CERT_PATH - Path to where certificate is saved (with auth --csr), - installed from, or revoked. (default: None) + Path to where certificate is saved (with certonly + --csr), installed from, or revoked (default: None) --key-path KEY_PATH Path to private key for certificate installation or revocation (if account key is missing) (default: None) --fullchain-path FULLCHAIN_PATH diff --git a/certbot/docs/compatibility.rst b/certbot/docs/compatibility.rst index a4f33c281..d94642ec6 100644 --- a/certbot/docs/compatibility.rst +++ b/certbot/docs/compatibility.rst @@ -21,9 +21,9 @@ may change at any time. The second is that Certbot's behavior should only be considered stable with certain files but not all. Files with which users should expect Certbot to maintain its current behavior with are: -* ``/etc/letsencrypt/live//{cert,chain,fullchain,privkey}.pem`` where - ```` is the name given to ``--cert-name``. If ``--cert-name`` is not - set by the user, it is the first domain given to ``--domains``. +* ``/etc/letsencrypt/live/$domain/{cert,chain,fullchain,privkey}.pem``, where + ``$domain`` is the certificate name (see :ref:`where-certs` + for more details) * :ref:`CLI configuration files ` * Hook directories in ``/etc/letsencrypt/renewal-hooks`` diff --git a/certbot/docs/conf.py b/certbot/docs/conf.py index 52820b69d..254bd3edd 100644 --- a/certbot/docs/conf.py +++ b/certbot/docs/conf.py @@ -97,7 +97,6 @@ language = None # directories to ignore when looking for source files. exclude_patterns = [ '_build', - 'man', 'challenges.rst', 'ciphers.rst' ] diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index def2c7fcd..13ac5ed68 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -478,13 +478,6 @@ to start contributing to Certbot. To run mypy on Certbot, use ``tox -e mypy`` on a machine that has Python 3 installed. -Note that instead of just importing ``typing``, due to packaging issues, in Certbot we import from -``acme.magic_typing`` and have to add some comments for pylint like this: - -.. code-block:: python - - from acme.magic_typing import Dict - Also note that OpenSSL, which we rely on, has type definitions for crypto but not SSL. We use both. Those imports should look like this: diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index 4366080e0..ac6c798eb 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -191,7 +191,7 @@ Optionally to install the Certbot Apache plugin, you can use: .. code-block:: shell - sudo apt-get install python-certbot-apache + sudo apt-get install python3-certbot-apache **Fedora** @@ -270,15 +270,10 @@ Disable and remove the swapfile once the virtual environment is constructed:: user@webserver:~$ sudo swapoff /tmp/swapfile user@webserver:~$ sudo rm /tmp/swapfile -Installing from source ----------------------- +Pip +--- -Installation from source is only supported for developers and the -whole process is described in the :doc:`contributing`. - -.. warning:: Please do **not** use ``python certbot/setup.py install``, ``python pip - install certbot``, or ``easy_install certbot``. Please do **not** attempt the - installation commands as superuser/root and/or without virtual environment, - e.g. ``sudo python certbot/setup.py install``, ``sudo pip install``, ``sudo - ./venv/bin/...``. These modes of operation might corrupt your operating - system and are **not supported** by the Certbot team! +Installing Certbot through pip is only supported on a best effort basis and +when using a virtual environment. Instructions for installing Certbot through +pip can be found at https://certbot.eff.org/instructions by selecting your +server software and then choosing "pip" in the "System" dropdown menu. diff --git a/certbot/docs/man/certbot.rst b/certbot/docs/man/certbot.rst index d03f3eed4..2f25504b0 100644 --- a/certbot/docs/man/certbot.rst +++ b/certbot/docs/man/certbot.rst @@ -1 +1,3 @@ +:orphan: + .. literalinclude:: ../cli-help.txt diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index ab8d64d79..87c4b9569 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -474,29 +474,37 @@ like Revoking certificates --------------------- -If your account key has been compromised or you otherwise need to revoke a certificate, -use the ``revoke`` command to do so. Note that the ``revoke`` command takes the certificate path -(ending in ``cert.pem``), not a certificate name or domain. Example:: +If you need to revoke a certificate, use the ``revoke`` subcommand to do so. - certbot revoke --cert-path /etc/letsencrypt/live/CERTNAME/cert.pem +A certificate may be revoked by providing its name (see ``certbot certificates``) or by providing +its path directly:: + + certbot revoke --cert-name example.com + + certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem + +If the certificate being revoked was obtained via the ``--staging``, ``--test-cert`` or a non-default ``--server`` flag, +that flag must be passed to the ``revoke`` subcommand. + +.. note:: After revocation, Certbot will (by default) ask whether you want to **delete** the certificate. + Unless deleted, Certbot will try to renew revoked certificates the next time ``certbot renew`` runs. You can also specify the reason for revoking your certificate by using the ``reason`` flag. Reasons include ``unspecified`` which is the default, as well as ``keycompromise``, ``affiliationchanged``, ``superseded``, and ``cessationofoperation``:: - certbot revoke --cert-path /etc/letsencrypt/live/CERTNAME/cert.pem --reason keycompromise + certbot revoke --cert-name example.com --reason keycompromise -Additionally, if a certificate -is a test certificate obtained via the ``--staging`` or ``--test-cert`` flag, that flag must be passed to the -``revoke`` subcommand. -Once a certificate is revoked (or for other certificate management tasks), all of a certificate's -relevant files can be removed from the system with the ``delete`` subcommand:: +Revoking by account key or certificate private key +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - certbot delete --cert-name example.com +By default, Certbot will try revoke the certificate using your ACME account key. If the certificate was created from +the same ACME account, the revocation will be successful. -.. note:: If you don't use ``delete`` to remove the certificate completely, it will be renewed automatically at the next renewal event. +If you instead have the corresponding private key file to the certificate you wish to revoke, use ``--key-path`` to perform the +revocation from any ACME account:: -.. note:: Revoking a certificate will have no effect on the rate limit imposed by the Let's Encrypt server. + certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem --key-path /etc/letsencrypt/live/example.com/privkey.pem .. _renewal: @@ -709,12 +717,24 @@ Where are my certificates? ========================== All generated keys and issued certificates can be found in -``/etc/letsencrypt/live/$domain``. In the case of creating a SAN certificate -with multiple alternative names, ``$domain`` is the first domain passed in -via -d parameter. Rather than copying, please point -your (web) server configuration directly to those files (or create -symlinks). During the renewal_, ``/etc/letsencrypt/live`` is updated -with the latest necessary files. +``/etc/letsencrypt/live/$domain``, where ``$domain`` is the certificate +name (see the note below). Rather than copying, please point your (web) +server configuration directly to those files (or create symlinks). +During the renewal_, ``/etc/letsencrypt/live`` is updated with the latest +necessary files. + +.. note:: + The certificate name ``$domain`` used in the path ``/etc/letsencrypt/live/$domain`` + follows this convention: + + * it is the name given to ``--cert-name``, + * if ``--cert-name`` is not set by the user it is the first domain given to + ``--domains``, + * if the first domain is a wildcard domain (eg. ``*.example.com``) the + certificate name will be ``example.com``, + * if a name collision would occur with a certificate already named ``example.com``, + the new certificate name will be constructed using a numerical sequence + as ``example.com-001``. For historical reasons, the containing directories are created with permissions of ``0700`` meaning that certificates are accessible only diff --git a/certbot/setup.py b/certbot/setup.py index 4ea98e574..8b2874f20 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -31,9 +31,9 @@ meta = dict(re.findall(r"""__([a-z]+)__ = '([^']+)""", read_file(init_fn))) readme = read_file(os.path.join(here, 'README.rst')) version = meta['version'] -# This package relies on PyOpenSSL, requests, and six, however, it isn't -# specified here to avoid masking the more specific request requirements in -# acme. See https://github.com/pypa/pip/issues/988 for more info. +# This package relies on PyOpenSSL and requests, however, it isn't specified +# here to avoid masking the more specific request requirements in acme. See +# https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme>=1.8.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index f9c218cd3..d0448a1db 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -276,14 +276,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() mock_open.side_effect = IOError - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertRaises( errors.AccountStorageError, self.storage.load, self.acc.id) def test_save_ioerrors(self): mock_open = mock.mock_open() mock_open.side_effect = IOError # TODO: [None, None, IOError] - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertRaises( errors.AccountStorageError, self.storage.save, self.acc, self.mock_client) diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index b26c1f624..ba6cfddc3 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -526,7 +526,7 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config): from certbot._internal.cert_manager import cert_path_to_lineage @@ -556,21 +556,21 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): mock_acceptable_matches.return_value = [lambda x: x.cert_path] test_cert_path = os.path.join(self.config.config_dir, 'live', 'example.org', 'cert.pem') - self.config.cert_path = (test_cert_path, '') + self.config.cert_path = test_cert_path self.assertEqual('example.org', self._call(self.config)) @mock.patch('certbot._internal.cert_manager._acceptable_matches') def test_options_archive_cert(self, mock_acceptable_matches): # Also this and the next test check that the regex of _archive_files is working. - self.config.cert_path = (os.path.join(self.config.config_dir, 'archive', 'example.org', - 'cert11.pem'), '') + self.config.cert_path = os.path.join(self.config.config_dir, 'archive', 'example.org', + 'cert11.pem') mock_acceptable_matches.return_value = [lambda x: self._archive_files(x, 'cert')] self.assertEqual('example.org', self._call(self.config)) @mock.patch('certbot._internal.cert_manager._acceptable_matches') def test_options_archive_fullchain(self, mock_acceptable_matches): - self.config.cert_path = (os.path.join(self.config.config_dir, 'archive', - 'example.org', 'fullchain11.pem'), '') + self.config.cert_path = os.path.join(self.config.config_dir, 'archive', + 'example.org', 'fullchain11.pem') mock_acceptable_matches.return_value = [lambda x: self._archive_files(x, 'fullchain')] self.assertEqual('example.org', self._call(self.config)) @@ -586,7 +586,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config, acceptable_matches, match_func, rv_func): from certbot._internal.cert_manager import match_and_check_overlaps @@ -595,7 +595,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): def test_basic_match(self): from certbot._internal.cert_manager import _acceptable_matches self.assertEqual(['example.org'], self._call(self.config, _acceptable_matches(), - lambda x: self.config.cert_path[0], lambda x: x.lineagename)) + lambda x: self.config.cert_path, lambda x: x.lineagename)) @mock.patch('certbot._internal.cert_manager._search_lineages') def test_no_matches(self, mock_search_lineages): diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index e65ec32ec..fca2b3e3e 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -1,16 +1,11 @@ """Tests for certbot._internal.cli.""" import argparse import copy +from importlib import reload as reload_module +import io import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six -from six.moves import reload_module # pylint: disable=import-error - from acme import challenges from certbot import errors from certbot._internal import cli @@ -21,6 +16,12 @@ from certbot.compat import os import certbot.tests.util as test_util from certbot.tests.util import TempDirTestCase +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + PLUGINS = disco.PluginsRegistry.find_all() @@ -91,7 +92,7 @@ class ParseTest(unittest.TestCase): def _help_output(self, args): "Run a command, and return the output string for scrutiny" - output = six.StringIO() + output = io.StringIO() def write_msg(message, *args, **kwargs): # pylint: disable=missing-docstring,unused-argument output.write(message) @@ -479,10 +480,6 @@ class ParseTest(unittest.TestCase): for topic in ['all', 'plugins', 'dns-route53']: self.assertFalse('certbot-route53:auth' in self._help_output([help_flag, topic])) - def test_no_permissions_check_accepted(self): - namespace = self.parse(["--no-permissions-check"]) - self.assertTrue(namespace.no_permissions_check) - class DefaultTest(unittest.TestCase): """Tests for certbot._internal.cli._Default.""" diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 0852ab175..4b08fe3a1 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -3,19 +3,20 @@ try: import readline # pylint: disable=import-error except ImportError: import certbot._internal.display.dummy_readline as readline # type: ignore +from importlib import reload as reload_module import string import sys import unittest +from certbot.compat import filesystem # pylint: disable=ungrouped-imports +from certbot.compat import os # pylint: disable=ungrouped-imports +import certbot.tests.util as test_util # pylint: disable=ungrouped-imports + try: import mock except ImportError: # pragma: no cover from unittest import mock -from six.moves import reload_module # pylint: disable=import-error -from certbot.compat import filesystem # pylint: disable=ungrouped-imports -from certbot.compat import os # pylint: disable=ungrouped-imports -import certbot.tests.util as test_util # pylint: disable=ungrouped-imports class CompleterTest(test_util.TempDirTestCase): @@ -29,7 +30,7 @@ class CompleterTest(test_util.TempDirTestCase): if self.tempdir[-1] != os.sep: self.tempdir += os.sep - self.paths = [] # type: List[str] + self.paths: List[str] = [] # create some files and directories in temp_dir for c in string.ascii_lowercase: path = os.path.join(self.tempdir, c) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 1b22d3422..30b33bbc9 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -1,20 +1,22 @@ """Test :mod:`certbot.display.util`.""" import inspect +import io import socket import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six from certbot import errors from certbot import interfaces from certbot.display import util as display_util import certbot.tests.util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + CHOICES = [("First", "Description1"), ("Second", "Description2")] TAGS = ["tag1", "tag2", "tag3"] TAGS_CHOICES = [("1", "tag1"), ("2", "tag2"), ("3", "tag3")] @@ -34,7 +36,7 @@ class InputWithTimeoutTest(unittest.TestCase): def test_input(self, prompt=None): expected = "foo bar" - stdin = six.StringIO(expected + "\n") + stdin = io.StringIO(expected + "\n") with mock.patch("certbot.compat.misc.select.select") as mock_select: mock_select.return_value = ([stdin], [], [],) self.assertEqual(self._call(prompt), expected) @@ -328,11 +330,7 @@ class FileOutputDisplayTest(unittest.TestCase): # Every IDisplay method implemented by FileDisplay must take # force_interactive to prevent workflow regressions. for name in interfaces.IDisplay.names(): - if six.PY2: - getargspec = inspect.getargspec - else: - getargspec = inspect.getfullargspec - arg_spec = getargspec(getattr(self.displayer, name)) # pylint: disable=deprecated-method + arg_spec = inspect.getfullargspec(getattr(self.displayer, name)) self.assertTrue("force_interactive" in arg_spec.args) @@ -402,12 +400,8 @@ class NoninteractiveDisplayTest(unittest.TestCase): for name in interfaces.IDisplay.names(): # pylint: disable=E1120 method = getattr(self.displayer, name) # asserts method accepts arbitrary keyword arguments - if six.PY2: - result = inspect.getargspec(method).keywords # pylint:deprecated-method - self.assertFalse(result is None) - else: - result = inspect.getfullargspec(method).varkw - self.assertFalse(result is None) + result = inspect.getfullargspec(method).varkw + self.assertFalse(result is None) class SeparateListInputTest(unittest.TestCase): diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 9ccd63a3a..265a42023 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -27,7 +27,7 @@ def set_signals(sig_handler_dict): def signal_receiver(signums): """Context manager to catch signals""" signals = [] - prev_handlers = get_signals(signums) # type: Dict[int, Union[int, None, Callable]] + prev_handlers: Dict[int, Union[int, None, Callable]] = get_signals(signums) set_signals({s: lambda s, _: signals.append(s) for s in signums}) yield signals set_signals(prev_handlers) diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 06a641216..b0522bd52 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -267,7 +267,7 @@ class RunSavedPostHooksTest(HookTest): def setUp(self): super(RunSavedPostHooksTest, self).setUp() - self.eventually = [] # type: List[str] + self.eventually: List[str] = [] def test_empty(self): self.assertFalse(self._call_with_mock_execute_and_eventually().called) diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 76764e61b..33d89c3ea 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -1,15 +1,11 @@ """Tests for certbot._internal.log.""" +import io import logging import logging.handlers import sys import time import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six from acme import messages from certbot import errors @@ -19,6 +15,12 @@ from certbot.compat import filesystem from certbot.compat import os from certbot.tests import util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + class PreArgParseSetupTest(unittest.TestCase): """Tests for certbot._internal.log.pre_arg_parse_setup.""" @@ -41,7 +43,7 @@ class PreArgParseSetupTest(unittest.TestCase): mock_root_logger.setLevel.assert_called_once_with(logging.DEBUG) self.assertEqual(mock_root_logger.addHandler.call_count, 2) - memory_handler = None # type: Optional[logging.handlers.MemoryHandler] + memory_handler: Optional[logging.handlers.MemoryHandler] = None for call in mock_root_logger.addHandler.call_args_list: handler = call[0][0] if memory_handler is None and isinstance(handler, logging.handlers.MemoryHandler): @@ -75,7 +77,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): self.devnull = open(os.devnull, 'w') from certbot._internal.log import ColoredStreamHandler - self.stream_handler = ColoredStreamHandler(six.StringIO()) + self.stream_handler = ColoredStreamHandler(io.StringIO()) from certbot._internal.log import MemoryHandler, TempHandler self.temp_handler = TempHandler() self.temp_path = self.temp_handler.path @@ -179,7 +181,7 @@ class ColoredStreamHandlerTest(unittest.TestCase): """Tests for certbot._internal.log.ColoredStreamHandler""" def setUp(self): - self.stream = six.StringIO() + self.stream = io.StringIO() self.stream.isatty = lambda: True self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) @@ -213,7 +215,7 @@ class MemoryHandlerTest(unittest.TestCase): self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.DEBUG) self.msg = 'hi there' - self.stream = six.StringIO() + self.stream = io.StringIO() self.stream_handler = logging.StreamHandler(self.stream) from certbot._internal.log import MemoryHandler @@ -238,7 +240,7 @@ class MemoryHandlerTest(unittest.TestCase): def test_target_reset(self): self._test_log_debug() - new_stream = six.StringIO() + new_stream = io.StringIO() new_stream_handler = logging.StreamHandler(new_stream) self.handler.setTarget(new_stream_handler) self.handler.flush(force=True) @@ -325,7 +327,7 @@ class PostArgParseExceptHookTest(unittest.TestCase): def test_acme_error(self): # Get an arbitrary error code - acme_code = next(six.iterkeys(messages.ERROR_CODES)) + acme_code = next(iter(messages.ERROR_CODES)) def get_acme_error(msg): """Wraps ACME errors so the constructor takes only a msg.""" @@ -349,7 +351,7 @@ class PostArgParseExceptHookTest(unittest.TestCase): def _test_common(self, error_type, debug): """Returns the mocked logger and stderr output.""" - mock_err = six.StringIO() + mock_err = io.StringIO() def write_err(*args, **unused_kwargs): """Write error to mock_err.""" diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index d1b4485ca..eaa20e6c9 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,9 +1,9 @@ # coding=utf-8 """Tests for certbot._internal.main.""" # pylint: disable=too-many-lines -from __future__ import print_function - import datetime +from importlib import reload as reload_module +import io import itertools import json from operator import is_ @@ -14,13 +14,7 @@ import traceback import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock import pytz -import six -from six.moves import reload_module # pylint: disable=import-error from certbot import crypto_util from certbot import errors @@ -40,6 +34,12 @@ from certbot.compat import os from certbot.plugins import enhancements import certbot.tests.util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.vector_path('cert_512.pem') @@ -286,10 +286,7 @@ class RevokeTest(test_util.TempDirTestCase): super(RevokeTest, self).setUp() shutil.copy(CERT_PATH, self.tempdir) - self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, - 'cert_512.pem')) - with open(self.tmp_cert_path, 'r') as f: - self.tmp_cert = (self.tmp_cert_path, f.read()) + self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, 'cert_512.pem')) patches = [ mock.patch('acme.client.BackwardsCompatibleClientV2'), @@ -319,6 +316,7 @@ class RevokeTest(test_util.TempDirTestCase): if not args: args = 'revoke --cert-path={0} ' args = args.format(self.tmp_cert_path).split() + cli.set_by_cli.detector = None # required to reset set_by_cli state plugins = disco.PluginsRegistry.find_all() config = configuration.NamespaceConfig( cli.prepare_and_parse_args(plugins, args)) @@ -344,13 +342,44 @@ class RevokeTest(test_util.TempDirTestCase): self.assertEqual(expected, mock_revoke.call_args_list) @mock.patch('certbot._internal.main._delete_if_appropriate') - @mock.patch('certbot._internal.storage.cert_path_for_cert_name') - def test_revoke_by_certname(self, mock_cert_path_for_cert_name, - mock_delete_if_appropriate): + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, + server="https://acme.example") args = 'revoke --cert-name=example.com'.split() - mock_cert_path_for_cert_name.return_value = self.tmp_cert mock_delete_if_appropriate.return_value = False self._call(args) + self.mock_acme_client.assert_called_once_with(mock.ANY, mock.ANY, 'https://acme.example') + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + + @mock.patch('certbot._internal.main._delete_if_appropriate') + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname_and_server(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + """Revoking with --server should use the server from the CLI""" + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, + server="https://acme.example") + args = 'revoke --cert-name=example.com --server https://other.example'.split() + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_acme_client.assert_called_once_with(mock.ANY, mock.ANY, 'https://other.example') + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + + @mock.patch('certbot._internal.main._delete_if_appropriate') + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname_empty_server(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + """Revoking with --cert-name where the lineage server is empty shouldn't crash """ + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, server=None) + args = 'revoke --cert-name=example.com'.split() + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_acme_client.assert_called_once_with( + mock.ANY, mock.ANY, constants.CLI_DEFAULTS['server']) self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) @mock.patch('certbot._internal.main._delete_if_appropriate') @@ -599,7 +628,7 @@ class MainTest(test_util.ConfigTestCase): "Run the client with output streams mocked out" args = self.standard_args + args - toy_stdout = stdout if stdout else six.StringIO() + toy_stdout = stdout if stdout else io.StringIO() with mock.patch('certbot._internal.main.sys.stdout', new=toy_stdout): with mock.patch('certbot._internal.main.sys.stderr') as stderr: with mock.patch("certbot.util.atexit"): @@ -612,8 +641,8 @@ class MainTest(test_util.ConfigTestCase): self.assertEqual(1, mock_run.call_count) def test_version_string_program_name(self): - toy_out = six.StringIO() - toy_err = six.StringIO() + toy_out = io.StringIO() + toy_err = io.StringIO() with mock.patch('certbot._internal.main.sys.stdout', new=toy_out): with mock.patch('certbot._internal.main.sys.stderr', new=toy_err): try: @@ -821,16 +850,16 @@ class MainTest(test_util.ConfigTestCase): flags = ['--init', '--prepare', '--authenticators', '--installers'] for args in itertools.chain( *(itertools.combinations(flags, r) - for r in six.moves.range(len(flags)))): + for r in range(len(flags)))): self._call(['plugins'] + list(args)) @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() - stdout = six.StringIO() + stdout = io.StringIO() with test_util.patch_get_utility_with_stdout(stdout=stdout): _, stdout, _, _ = self._call(['plugins'], stdout) @@ -842,7 +871,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args_unprivileged(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() def throw_error(directory, mode, strict): @@ -850,7 +879,7 @@ class MainTest(test_util.ConfigTestCase): _, _, _ = directory, mode, strict raise errors.Error() - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('certbot.util.set_up_core_dir') as mock_set_up_core_dir: with test_util.patch_get_utility_with_stdout(stdout=stdout): mock_set_up_core_dir.side_effect = throw_error @@ -864,10 +893,10 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_init(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() - stdout = six.StringIO() + stdout = io.StringIO() with test_util.patch_get_utility_with_stdout(stdout=stdout): _, stdout, _, _ = self._call(['plugins', '--init'], stdout) @@ -882,10 +911,10 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_prepare(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() - stdout = six.StringIO() + stdout = io.StringIO() with test_util.patch_get_utility_with_stdout(stdout=stdout): _, stdout, _, _ = self._call(['plugins', '--init', '--prepare'], stdout) @@ -1037,7 +1066,7 @@ class MainTest(test_util.ConfigTestCase): mock_certr = mock.MagicMock() mock_key = mock.MagicMock(pem='pem_key') mock_client = mock.MagicMock() - stdout = six.StringIO() + stdout = io.StringIO() mock_client.obtain_certificate.return_value = (mock_certr, 'chain', mock_key, 'csr') @@ -1055,7 +1084,7 @@ class MainTest(test_util.ConfigTestCase): mock_get_utility().notification.side_effect = write_msg with mock.patch('certbot._internal.main.renewal.OpenSSL') as mock_ssl: mock_latest = mock.MagicMock() - mock_latest.get_issuer.return_value = "Fake fake" + mock_latest.get_issuer.return_value = "Artificial pretend" mock_ssl.crypto.load_certificate.return_value = mock_latest with mock.patch('certbot._internal.main.renewal.crypto_util') \ as mock_crypto_util: @@ -1326,7 +1355,7 @@ class MainTest(test_util.ConfigTestCase): _, _, stdout = self._test_renewal_common( due_for_renewal=False, extra_args=None, should_renew=False, args=['renew', '--post-hook', - '{0} -c "from __future__ import print_function; print(\'hello world\');"' + '{0} -c "print(\'hello world\');"' .format(sys.executable)]) self.assertTrue('No hooks were run.' in stdout.getvalue()) diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index ed13544de..64829c8cc 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -8,7 +8,6 @@ try: except ImportError: # pragma: no cover from unittest import mock import pkg_resources -import six import zope.interface from certbot import errors @@ -56,7 +55,7 @@ class PluginEntryPointTest(unittest.TestCase): EP_SA: "sa", } - for entry_point, name in six.iteritems(names): + for entry_point, name in names.items(): self.assertEqual( name, PluginEntryPoint.entry_point_to_plugin_name(entry_point, with_prefix=False)) @@ -70,7 +69,7 @@ class PluginEntryPointTest(unittest.TestCase): self.ep3: "p3:ep3", } - for entry_point, name in six.iteritems(names): + for entry_point, name in names.items(): self.assertEqual( name, PluginEntryPoint.entry_point_to_plugin_name(entry_point, with_prefix=True)) @@ -278,7 +277,7 @@ class PluginsRegistryTest(unittest.TestCase): self.plugin_ep.prepare.assert_called_once_with() def test_prepare_order(self): - order = [] # type: List[str] + order: List[str] = [] plugins = dict( (c, mock.MagicMock(prepare=functools.partial(order.append, c))) for c in string.ascii_letters) diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index 993f3b461..31761e986 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -29,7 +29,7 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen def more_info(self): # pylint: disable=missing-docstring,no-self-use return 'A fake authenticator for testing.' - class _FakeConfig(object): + class _FakeConfig: fake_propagation_seconds = 0 fake_config_key = 1 fake_other_key = None diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 7318783fd..0e552e0c5 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -6,7 +6,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six from acme import challenges from certbot import errors @@ -53,7 +52,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.assertRaises(errors.HookCommandNotFound, self.auth.prepare) def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) + self.assertTrue(isinstance(self.auth.more_info(), str)) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref('example.org'), @@ -61,7 +60,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): def test_script_perform(self): self.config.manual_auth_hook = ( - '{0} -c "from __future__ import print_function;' + '{0} -c "' 'from certbot.compat import os;' 'print(os.environ.get(\'CERTBOT_DOMAIN\'));' 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index 47708e340..dad5b270a 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -5,7 +5,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six class InstallerTest(unittest.TestCase): @@ -16,7 +15,7 @@ class InstallerTest(unittest.TestCase): self.installer = Installer(config=mock.MagicMock(), name="null") def test_it(self): - self.assertTrue(isinstance(self.installer.more_info(), six.string_types)) + self.assertTrue(isinstance(self.installer.more_info(), str)) self.assertEqual([], self.installer.get_all_names()) self.assertEqual([], self.installer.supported_enhancements()) diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index a5de99e60..027ebdc35 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -52,7 +52,7 @@ class PickPluginTest(unittest.TestCase): self.default = None self.reg = mock.MagicMock() self.question = "Question?" - self.ifaces = [] # type: List[interfaces.IPlugin] + self.ifaces: List[interfaces.IPlugin] = [] def _call(self): from certbot._internal.plugins.selection import pick_plugin diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 751b9d943..80347d35a 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -10,7 +10,6 @@ try: except ImportError: # pragma: no cover from unittest import mock import OpenSSL.crypto # pylint: disable=unused-import -import six from acme import challenges from acme import standalone as acme_standalone # pylint: disable=unused-import @@ -25,9 +24,8 @@ class ServerManagerTest(unittest.TestCase): def setUp(self): from certbot._internal.plugins.standalone import ServerManager - self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] - self.http_01_resources = {} \ - # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] + self.certs: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] = {} + self.http_01_resources: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] = {} self.mgr = ServerManager(self.certs, self.http_01_resources) def test_init(self): @@ -91,7 +89,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.servers = mock.MagicMock() def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) + self.assertTrue(isinstance(self.auth.more_info(), str)) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref(domain=None), diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index 4b0d1da83..2999d306e 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -33,7 +33,7 @@ class PluginStorageTest(test_util.ConfigTestCase): mock_open.side_effect = IOError self.plugin.storage.storagepath = os.path.join(self.config.config_dir, ".pluginstorage.json") - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): with mock.patch('certbot.compat.os.path.isfile', return_value=True): with mock.patch("certbot.reverter.util"): self.assertRaises(errors.PluginStorageError, diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index e57e09eae..e6fbd8e88 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -14,7 +14,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six from acme import challenges from certbot import achallenges @@ -59,7 +58,7 @@ class AuthenticatorTest(unittest.TestCase): def test_more_info(self): more_info = self.auth.more_info() - self.assertTrue(isinstance(more_info, six.string_types)) + self.assertTrue(isinstance(more_info, str)) self.assertTrue(self.path in more_info) def test_add_parser_arguments(self): @@ -83,7 +82,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertTrue(self.achall.domain in call[0][0]) self.assertTrue(all( webroot in call[0][1] - for webroot in six.itervalues(self.config.webroot_map))) + for webroot in self.config.webroot_map.values())) self.assertEqual(self.config.webroot_map[self.achall.domain], self.path) @@ -100,7 +99,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertTrue(self.achall.domain in call[0][0]) self.assertTrue(all( webroot in call[0][1] - for webroot in six.itervalues(self.config.webroot_map))) + for webroot in self.config.webroot_map.values())) @test_util.patch_get_utility() def test_new_webroot(self, mock_get_utility): diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 7d03f1821..7a37f782e 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -1,12 +1,13 @@ """Tests for certbot._internal.reporter.""" +import io import sys import unittest + try: import mock except ImportError: # pragma: no cover from unittest import mock -import six class ReporterTest(unittest.TestCase): @@ -16,7 +17,7 @@ class ReporterTest(unittest.TestCase): self.reporter = reporter.Reporter(mock.MagicMock(quiet=False)) self.old_stdout = sys.stdout # type: ignore - sys.stdout = six.StringIO() + sys.stdout = io.StringIO() def tearDown(self): sys.stdout = self.old_stdout diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index d67aa431a..af01a9a1b 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -9,7 +9,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six from certbot import errors from certbot.compat import os @@ -156,7 +155,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): act_coms = get_undo_commands(self.config.temp_checkpoint_dir) - for a_com, com in six.moves.zip(act_coms, coms): + for a_com, com in zip(act_coms, coms): self.assertEqual(a_com, com) def test_bad_register_undo_command(self): diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 914304cd4..1e3a1ffd8 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -11,7 +11,6 @@ try: except ImportError: # pragma: no cover from unittest import mock import pytz -import six import certbot from certbot import errors @@ -286,7 +285,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version("cert"), None) def test_latest_and_next_versions(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(self.test_rc.latest_common_version(), 5) @@ -326,7 +325,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_update_link_to(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) @@ -353,12 +352,12 @@ class RenewableCertTests(BaseRenewableCertTest): os.path.basename(self.test_rc.version("cert", 8))) def test_update_all_links_to_success(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) self.assertEqual(self.test_rc.latest_common_version(), 5) - for ver in six.moves.range(1, 6): + for ver in range(1, 6): self.test_rc.update_all_links_to(ver) for kind in ALL_FOUR: self.assertEqual(ver, self.test_rc.current_version(kind)) @@ -396,11 +395,11 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version(kind), 11) def test_has_pending_deployment(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) - for ver in six.moves.range(1, 6): + for ver in range(1, 6): self.test_rc.update_all_links_to(ver) for kind in ALL_FOUR: self.assertEqual(ver, self.test_rc.current_version(kind)) @@ -498,7 +497,7 @@ class RenewableCertTests(BaseRenewableCertTest): # (to avoid instantiating parser) mock_rv.side_effect = lambda x: x - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.test_rc.update_all_links_to(3) @@ -529,7 +528,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.version("privkey", i)))) for kind in ALL_FOUR: - self.assertEqual(self.test_rc.available_versions(kind), list(six.moves.range(1, 9))) + self.assertEqual(self.test_rc.available_versions(kind), list(range(1, 9))) self.assertEqual(self.test_rc.current_version(kind), 3) # Test updating from latest version rather than old version self.test_rc.update_all_links_to(8) @@ -538,7 +537,7 @@ class RenewableCertTests(BaseRenewableCertTest): b'attempt', self.config)) for kind in ALL_FOUR: self.assertEqual(self.test_rc.available_versions(kind), - list(six.moves.range(1, 10))) + list(range(1, 10))) self.assertEqual(self.test_rc.current_version(kind), 8) with open(self.test_rc.version("fullchain", 9)) as f: self.assertEqual(f.read(), "last" + "attempt") @@ -766,6 +765,13 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(storage.add_time_interval(base_time, interval), excepted) + def test_server(self): + self.test_rc.configuration["renewalparams"] = {} + self.assertEqual(self.test_rc.server, None) + rp = self.test_rc.configuration["renewalparams"] + rp["server"] = "https://acme.example/dir" + self.assertEqual(self.test_rc.server, "https://acme.example/dir") + def test_is_test_cert(self): self.test_rc.configuration["renewalparams"] = {} rp = self.test_rc.configuration["renewalparams"] @@ -935,14 +941,14 @@ class CertPathForCertNameTest(BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config, certname): from certbot._internal.storage import cert_path_for_cert_name return cert_path_for_cert_name(cli_config, certname) def test_simple_cert_name(self): - self.assertEqual(self._call(self.config, 'example.org'), (self.fullchain, 'fullchain')) + self.assertEqual(self._call(self.config, 'example.org'), self.fullchain) def test_no_such_cert_name(self): self.assertRaises(errors.CertStorageError, self._call, self.config, 'fake-example.org') diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 7b510fbb6..18947c342 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -1,21 +1,22 @@ """Tests for certbot.util.""" import argparse import errno +from importlib import reload as reload_module +import io import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six -from six.moves import reload_module # pylint: disable=import-error - from certbot import errors from certbot.compat import filesystem from certbot.compat import os import certbot.tests.util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + class EnvNoSnapForExternalCallsTest(unittest.TestCase): """Tests for certbot.util.env_no_snap_for_external_calls.""" @@ -265,11 +266,11 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): def test_multiple(self): items = [] - for _ in six.moves.range(10): + for _ in range(10): items.append(self._call("wow")) f, name = items[-1] self.assertTrue(isinstance(f, file_type)) - self.assertTrue(isinstance(name, six.string_types)) + self.assertTrue(isinstance(name, str)) self.assertTrue("wow-0009.conf" in name) for f, _ in items: f.close() @@ -361,7 +362,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): def test_help(self): self._call("--old-option", 2) - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch("sys.stdout", new=stdout): try: self.parser.parse_args(["-h"]) diff --git a/letsencrypt-auto b/letsencrypt-auto index 002fd5ffc..24af0b4c3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.12.0" +LE_AUTO_VERSION="1.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -806,6 +806,7 @@ elif [ -f /etc/mageia-release ]; then NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -1487,18 +1488,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index aba5f1140..fbd16f2b0 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmAZorcACgkQTRfJlc2X -dfI6Ogf+LFASyH9sgTV1k9hs1zbmO3CxyE9QQs1JLXpoKOQ1tKv+v+kpt+lJ005g -rielyRSssXtZSyfLchCSBh6qaEBodoOcz8RS2z7rDnR9jKOJv252Buh2oSa3KPmn -WPjRmB3zVXnhq/XmPKQTnoflUlBg+MtZuZXt0Fvu8rvQB+RY3AUfB5Xs83nxJNj4 -W9qNpZYl0sJWWiydr23bEk35MJSt62sKDvyqIVjUfgDfXHmauOpg0foz2xS6XP8i -Ke66GUKaQ1ap2BTucwVT0hieXiQZpxx1PitUeEOjOH9PUfrAxyFlQ0XQaVlqoBhc -YM3nzJw9yf12b+XCUvMzHyQmDA5vdQ== -=AUGt +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmA+sv4ACgkQTRfJlc2X +dfLG+ggAvwUCqy06UZd8jZhOZFUoi8nWHG2TMnm/4CW4G9PPCjsCoQplhaAaUCrR +PAv0vtsG1rBKWekICg6IBTzLVioH9xRPUkpfbVQhT1c1T/5CqMsFFXR5p9YAKRe7 +hlOb7VRN10bdCS4JThRPNhdWFdWKZXYKcIOObWA/FX2GacxMHuLwPpSsbt2NRffy +qz1ZOWxvr289aWEbZWfyBiI2bxQ7wlMEbZ/JLUXDe46ETDxzENu+c0x2109ha6m4 +wHmUS0r16ps/n9DueTZGJf3C26mU+cIB+LgNvOcibBo+0Ly7t+OiBHYbkFXS2KTQ +RbH4bPZrsduUOzhE8wIsSUIsVGDleQ== +=jYnL -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 14e71c615..b0b25759f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.13.0.dev0" +LE_AUTO_VERSION="1.14.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -800,12 +800,14 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -1487,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index ac143de51..3516f77b0 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 783268571..72876bb59 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -322,12 +322,14 @@ DeterminePythonVersion() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 4d4c91a5d..0f51c4b8a 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index c9061ecb3..d53fba88b 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -73,10 +73,10 @@ parts: build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev] build-environment: - SNAPCRAFT_PYTHON_VENV_ARGS: --upgrade - # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the + # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the # parts.[part_name].constraints option available in snapcraft.yaml when the Python plugin is # used. This is done to let these constraints be applied not only on the certbot package - # build, but also on any isolated build that pip could trigger when building wheels for + # build, but also on any isolated build that pip could trigger when building wheels for # dependencies. See https://github.com/certbot/certbot/pull/8443 for more info. - PIP_CONSTRAINT: $SNAPCRAFT_PART_SRC/snap-constraints.txt override-build: | @@ -85,7 +85,7 @@ parts: snapcraftctl build override-pull: | snapcraftctl pull - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/letsencrypt-auto-source/pieces/dependency-requirements.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_constraints.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` diff --git a/tests/letstest/auto_targets.yaml b/tests/letstest/auto_targets.yaml index 164580e86..01d410227 100644 --- a/tests/letstest/auto_targets.yaml +++ b/tests/letstest/auto_targets.yaml @@ -57,10 +57,3 @@ targets: type: centos virt: hvm user: centos - #----------------------------------------------------------------------------- - # Amazon Linux - - ami: ami-0ff8a91507f77f867 - name: amazon - type: centos - virt: hvm - user: ec2-user diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 5ad1d8c15..2e4399000 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -27,10 +27,6 @@ see: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html """ - -from __future__ import print_function -from __future__ import with_statement - import argparse import multiprocessing as mp from multiprocessing import Manager @@ -39,17 +35,16 @@ import socket import sys import time import traceback +import urllib.error as urllib_error +import urllib.request as urllib_request import boto3 from botocore.exceptions import ClientError -from six.moves.urllib import error as urllib_error -from six.moves.urllib import request as urllib_request import yaml from fabric import Config from fabric import Connection - # Command line parser #------------------------------------------------------------------------------- parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.') @@ -95,7 +90,7 @@ SECURITY_GROUP_NAME = 'certbot-security-group' SENTINEL = None #queue kill signal SUBNET_NAME = 'certbot-subnet' -class Status(object): +class Status: """Possible statuses of client tests.""" PASS = 'pass' FAIL = 'fail' @@ -195,18 +190,16 @@ def block_until_ssh_open(ipstring, wait_time=10, timeout=120): t_elapsed += wait_time sock.close() -def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20): +def block_until_instance_ready(booting_instance, extra_wait_time=20): "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections" - state = booting_instance.state['Name'] - ip = booting_instance.public_ip_address - while state != 'running' or ip is None: - time.sleep(wait_time) - # The instance needs to be reloaded to update its local attributes. See - # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Instance.reload. - booting_instance.reload() - state = booting_instance.state['Name'] - ip = booting_instance.public_ip_address - block_until_ssh_open(ip) + booting_instance.wait_until_running() + # The instance needs to be reloaded to update its local attributes. See + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Instance.reload. + booting_instance.reload() + # After waiting for the instance to be running and reloading the instance + # state, we should have an IP address. + assert booting_instance.public_ip_address is not None + block_until_ssh_open(booting_instance.public_ip_address) time.sleep(extra_wait_time) return booting_instance diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index d0b941736..3964364e1 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -153,17 +153,8 @@ if ! ./letsencrypt-auto -v --debug --version 2>&1 | grep "will no longer receive exit 1 fi -# Finally, we check if our local server received more requests. Over time, -# we'll move more and more OSes into this case until it this is the expected -# behavior on all systems. -if [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue; then - if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then - echo our local server received unexpected requests - exit 1 - fi -else - if diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then - echo our local server did not receive the requests we expected - exit 1 - fi +# Finally, we check if our local server received more requests. +if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then + echo our local server received unexpected requests + exit 1 fi diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index aa12d5610..becdd6d9a 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -12,13 +12,21 @@ sudo $BOOTSTRAP_SCRIPT # We strip the hashes because the venv creation script includes unhashed # constraints in the commands given to pip and the mix of hashed and unhashed # packages makes pip error out. -python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > requirements.txt -# We also strip out the requirement for enum34 because it cannot be installed -# in newer versions of Python 3, tools/strip_hashes.py removes the environment -# marker that'd normally prevent it from being installed, and this package is -# not needed for any OS tested here. -sed -i '/enum34/d' requirements.txt -CERTBOT_PIP_NO_BINARY=:all: tools/venv.py --requirement requirements.txt +python3 tools/strip_hashes.py tools/pipstrap_constraints.txt > constraints.txt +python3 tools/strip_hashes.py tools/certbot_constraints.txt > requirements.txt + +# We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7 +# because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been +# dropped on cryptography>=3.2 and pyOpenSSL>=20.0.0. +# Using this old version of OpenSSL would break the cryptography and pyOpenSSL wheels builds. +if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then + sed -i 's|cryptography==.*|cryptography==3.1.1|g' requirements.txt + sed -i 's|pyOpenSSL==.*|pyOpenSSL==19.1.0|g' requirements.txt +fi + +python3 -m venv $VENV_PATH +$VENV_PATH/bin/python3 tools/pipstrap.py +PIP_CONSTRAINT=constraints.txt PIP_NO_BINARY=:all: $VENV_PATH/bin/python3 -m pip install --requirement requirements.txt . "$VENV_PATH/bin/activate" # pytest is needed to run tests on some of our packages so we install a pinned version here. tools/pip_install.py pytest diff --git a/tests/lock_test.py b/tests/lock_test.py index 56399c874..f310b5753 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -1,6 +1,4 @@ """Tests to ensure the lock order is preserved.""" -from __future__ import print_function - import atexit import datetime import functools diff --git a/tests/modification-check.py b/tests/modification-check.py index 7a69fb1db..357b25350 100755 --- a/tests/modification-check.py +++ b/tests/modification-check.py @@ -7,16 +7,13 @@ import shutil import subprocess import sys import tempfile +from urllib.request import urlretrieve -try: - from urllib.request import urlretrieve -except ImportError: - from urllib import urlretrieve def find_repo_path(): return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -# We do not use filecmp.cmp to take advantage of universal newlines +# We do not use filecmp.cmp to take advantage of universal newlines # handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. # As a consequence, this function will not work correctly if executed by Python 2.x on Windows. # But it will work correctly on Linux for any version, because every file tested will be LF. @@ -50,10 +47,10 @@ def validate_scripts_content(repo_path, temp_cwd): errors = True else: shutil.copyfile( - os.path.join(repo_path, 'certbot-auto'), + os.path.join(repo_path, 'certbot-auto'), os.path.join(temp_cwd, 'local-auto')) shutil.copy(os.path.normpath(os.path.join( - repo_path, + repo_path, 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) # Compare file against current version in the target branch @@ -72,7 +69,7 @@ def validate_scripts_content(repo_path, temp_cwd): latest_version = subprocess.check_output( [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) subprocess.check_call( - [sys.executable, 'fetch.py', '--le-auto-script', + [sys.executable, 'fetch.py', '--le-auto-script', 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) if compare_files( os.path.join(temp_cwd, 'letsencrypt-auto'), diff --git a/tools/certbot_constraints.txt b/tools/certbot_constraints.txt new file mode 100644 index 000000000..77bfef9db --- /dev/null +++ b/tools/certbot_constraints.txt @@ -0,0 +1,262 @@ +# This is the flattened list of pinned packages to build certbot deployable artifacts. +# To generate this, do (with docker and package hashin installed): +# ``` +# tools/rebuild_certbot_contraints.py \ +# tools/certbot_constraints.txt +# ``` +# If you want to update a single dependency, run commands similar to these: +# ``` +# pip install hashin +# hashin -r dependency-requirements.txt cryptography==1.5.2 +# ``` +ConfigArgParse==1.2.3 \ + --hash=sha256:edd17be986d5c1ba2e307150b8e5f5107aba125f3574dddd02c85d5cdcfd37dc +certifi==2020.12.5 \ + --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ + --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 +cffi==1.14.4 \ + --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \ + --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \ + --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \ + --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \ + --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \ + --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \ + --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \ + --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \ + --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \ + --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \ + --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \ + --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \ + --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \ + --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \ + --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \ + --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \ + --hash=sha256:7ef7d4ced6b325e92eb4d3502946c78c5367bc416398d387b39591532536734e \ + --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \ + --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \ + --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \ + --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \ + --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \ + --hash=sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01 \ + --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \ + --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \ + --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \ + --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \ + --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \ + --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \ + --hash=sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e \ + --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \ + --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \ + --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \ + --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \ + --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \ + --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \ + --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f +chardet==4.0.0 \ + --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ + --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 +configobj==5.0.6 \ + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 +cryptography==3.3.2 \ + --hash=sha256:0d7b69674b738068fa6ffade5c962ecd14969690585aaca0a1b1fc9058938a72 \ + --hash=sha256:1bd0ccb0a1ed775cd7e2144fe46df9dc03eefd722bbcf587b3e0616ea4a81eff \ + --hash=sha256:3c284fc1e504e88e51c428db9c9274f2da9f73fdf5d7e13a36b8ecb039af6e6c \ + --hash=sha256:49570438e60f19243e7e0d504527dd5fe9b4b967b5a1ff21cc12b57602dd85d3 \ + --hash=sha256:541dd758ad49b45920dda3b5b48c968f8b2533d8981bcdb43002798d8f7a89ed \ + --hash=sha256:5a60d3780149e13b7a6ff7ad6526b38846354d11a15e21068e57073e29e19bed \ + --hash=sha256:7951a966613c4211b6612b0352f5bf29989955ee592c4a885d8c7d0f830d0433 \ + --hash=sha256:922f9602d67c15ade470c11d616f2b2364950602e370c76f0c94c94ae672742e \ + --hash=sha256:a0f0b96c572fc9f25c3f4ddbf4688b9b38c69836713fb255f4a2715d93cbaf44 \ + --hash=sha256:a777c096a49d80f9d2979695b835b0f9c9edab73b59e4ceb51f19724dda887ed \ + --hash=sha256:a9a4ac9648d39ce71c2f63fe7dc6db144b9fa567ddfc48b9fde1b54483d26042 \ + --hash=sha256:aa4969f24d536ae2268c902b2c3d62ab464b5a66bcb247630d208a79a8098e9b \ + --hash=sha256:c7390f9b2119b2b43160abb34f63277a638504ef8df99f11cb52c1fda66a2e6f \ + --hash=sha256:e18e6ab84dfb0ab997faf8cca25a86ff15dfea4027b986322026cc99e0a892da +distro==1.5.0 \ + --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ + --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 +idna==2.10 \ + --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ + --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 +josepy==1.6.0 \ + --hash=sha256:0aab1c3ceffe045e7fd5bcfe7685e27e9d2758518d9ba7116b5de34087e70bf5 \ + --hash=sha256:65f077fc5902aca1e140ddb000e7abb081d5fb8421db60b6071076ef81c5bd27 +parsedatetime==2.6 \ + --hash=sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455 \ + --hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b +pyOpenSSL==20.0.1 \ + --hash=sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51 \ + --hash=sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b +pyRFC3339==1.1 \ + --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ + --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a +pycparser==2.20 \ + --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ + --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 +pyparsing==2.4.7 \ + --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ + --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b +python-augeas==0.5.0 \ + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 +pytz==2021.1 \ + --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \ + --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 +requests==2.25.1 \ + --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ + --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e +requests-toolbelt==0.9.1 \ + --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ + --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced +urllib3==1.26.3 \ + --hash=sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80 \ + --hash=sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73 +zope.component==4.6.2 \ + --hash=sha256:607628e4c84f7887a69a958542b5c304663e726b73aba0882e3a3f059bff14f3 \ + --hash=sha256:91628918218b3e6f6323de2a7b845e09ddc5cae131c034896c051b084bba3c92 +zope.deferredimport==4.3.1 \ + --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \ + --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a +zope.deprecation==4.4.0 \ + --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ + --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 +zope.event==4.5.0 \ + --hash=sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42 \ + --hash=sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330 +zope.hookable==5.0.1 \ + --hash=sha256:0194b9b9e7f614abba60c90b231908861036578297515d3d6508eb10190f266d \ + --hash=sha256:0c2977473918bdefc6fa8dfb311f154e7f13c6133957fe649704deca79b92093 \ + --hash=sha256:17b8bdb3b77e03a152ca0d5ca185a7ae0156f5e5a2dbddf538676633a1f7380f \ + --hash=sha256:29d07681a78042cdd15b268ae9decffed9ace68a53eebeb61d65ae931d158841 \ + --hash=sha256:36fb1b35d1150267cb0543a1ddd950c0bc2c75ed0e6e92e3aaa6ac2e29416cb7 \ + --hash=sha256:3aed60c2bb5e812bbf9295c70f25b17ac37c233f30447a96c67913ba5073642f \ + --hash=sha256:3cac1565cc768911e72ca9ec4ddf5c5109e1fef0104f19f06649cf1874943b60 \ + --hash=sha256:3d4bc0cc4a37c3cd3081063142eeb2125511db3c13f6dc932d899c512690378e \ + --hash=sha256:3f73096f27b8c28be53ffb6604f7b570fbbb82f273c6febe5f58119009b59898 \ + --hash=sha256:522d1153d93f2d48aa0bd9fb778d8d4500be2e4dcf86c3150768f0e3adbbc4ef \ + --hash=sha256:523d2928fb7377bbdbc9af9c0b14ad73e6eaf226349f105733bdae27efd15b5a \ + --hash=sha256:5848309d4fc5c02150a45e8f8d2227e5bfda386a508bbd3160fed7c633c5a2fa \ + --hash=sha256:6781f86e6d54a110980a76e761eb54590630fd2af2a17d7edf02a079d2646c1d \ + --hash=sha256:6fd27921ebf3aaa945fa25d790f1f2046204f24dba4946f82f5f0a442577c3e9 \ + --hash=sha256:70d581862863f6bf9e175e85c9d70c2d7155f53fb04dcdb2f73cf288ca559a53 \ + --hash=sha256:81867c23b0dc66c8366f351d00923f2bc5902820a24c2534dfd7bf01a5879963 \ + --hash=sha256:81db29edadcbb740cd2716c95a297893a546ed89db1bfe9110168732d7f0afdd \ + --hash=sha256:86bd12624068cea60860a0759af5e2c3adc89c12aef6f71cf12f577e28deefe3 \ + --hash=sha256:9c184d8f9f7a76e1ced99855ccf390ffdd0ec3765e5cbf7b9cada600accc0a1e \ + --hash=sha256:acc789e8c29c13555e43fe4bf9fcd15a65512c9645e97bbaa5602e3201252b02 \ + --hash=sha256:afaa740206b7660d4cc3b8f120426c85761f51379af7a5b05451f624ad12b0af \ + --hash=sha256:b5f5fa323f878bb16eae68ea1ba7f6c0419d4695d0248bed4b18f51d7ce5ab85 \ + --hash=sha256:bd89e0e2c67bf4ac3aca2a19702b1a37269fb1923827f68324ac2e7afd6e3406 \ + --hash=sha256:c212de743283ec0735db24ec6ad913758df3af1b7217550ff270038062afd6ae \ + --hash=sha256:ca553f524293a0bdea05e7f44c3e685e4b7b022cb37d87bc4a3efa0f86587a8d \ + --hash=sha256:cab67065a3db92f636128d3157cc5424a145f82d96fb47159c539132833a6d36 \ + --hash=sha256:d3b3b3eedfdbf6b02898216e85aa6baf50207f4378a2a6803d6d47650cd37031 \ + --hash=sha256:d9f4a5a72f40256b686d31c5c0b1fde503172307beb12c1568296e76118e402c \ + --hash=sha256:df5067d87aaa111ed5d050e1ee853ba284969497f91806efd42425f5348f1c06 \ + --hash=sha256:e2587644812c6138f05b8a41594a8337c6790e3baf9a01915e52438c13fc6bef \ + --hash=sha256:e27fd877662db94f897f3fd532ef211ca4901eb1a70ba456f15c0866a985464a \ + --hash=sha256:e427ebbdd223c72e06ba94c004bb04e996c84dec8a0fa84e837556ae145c439e \ + --hash=sha256:e583ad4309c203ef75a09d43434cf9c2b4fa247997ecb0dcad769982c39411c7 \ + --hash=sha256:e760b2bc8ece9200804f0c2b64d10147ecaf18455a2a90827fbec4c9d84f3ad5 \ + --hash=sha256:ea9a9cc8bcc70e18023f30fa2f53d11ae069572a162791224e60cd65df55fb69 \ + --hash=sha256:ecb3f17dce4803c1099bd21742cd126b59817a4e76a6544d31d2cca6e30dbffd \ + --hash=sha256:ed794e3b3de42486d30444fb60b5561e724ee8a2d1b17b0c2e0f81e3ddaf7a87 \ + --hash=sha256:ee885d347279e38226d0a437b6a932f207f691c502ee565aba27a7022f1285df \ + --hash=sha256:fd5e7bc5f24f7e3d490698f7b854659a9851da2187414617cd5ed360af7efd63 \ + --hash=sha256:fe45f6870f7588ac7b2763ff1ce98cce59369717afe70cc353ec5218bc854bcc +zope.interface==5.2.0 \ + --hash=sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1 \ + --hash=sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d \ + --hash=sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123 \ + --hash=sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232 \ + --hash=sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549 \ + --hash=sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102 \ + --hash=sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5 \ + --hash=sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45 \ + --hash=sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00 \ + --hash=sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc \ + --hash=sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7 \ + --hash=sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104 \ + --hash=sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034 \ + --hash=sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3 \ + --hash=sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3 \ + --hash=sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4 \ + --hash=sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86 \ + --hash=sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96 \ + --hash=sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546 \ + --hash=sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb \ + --hash=sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3 \ + --hash=sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b \ + --hash=sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b \ + --hash=sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec \ + --hash=sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae \ + --hash=sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e \ + --hash=sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386 \ + --hash=sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2 \ + --hash=sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a \ + --hash=sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d \ + --hash=sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a \ + --hash=sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24 \ + --hash=sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d \ + --hash=sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b \ + --hash=sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50 \ + --hash=sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523 \ + --hash=sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a \ + --hash=sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095 \ + --hash=sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a \ + --hash=sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520 \ + --hash=sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65 \ + --hash=sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11 \ + --hash=sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c \ + --hash=sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7 \ + --hash=sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332 \ + --hash=sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e \ + --hash=sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c \ + --hash=sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7 \ + --hash=sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20 \ + --hash=sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc \ + --hash=sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd \ + --hash=sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537 +zope.proxy==4.3.5 \ + --hash=sha256:00573dfa755d0703ab84bb23cb6ecf97bb683c34b340d4df76651f97b0bab068 \ + --hash=sha256:092049280f2848d2ba1b57b71fe04881762a220a97b65288bcb0968bb199ec30 \ + --hash=sha256:0cbd27b4d3718b5ec74fc65ffa53c78d34c65c6fd9411b8352d2a4f855220cf1 \ + --hash=sha256:17fc7e16d0c81f833a138818a30f366696653d521febc8e892858041c4d88785 \ + --hash=sha256:19577dfeb70e8a67249ba92c8ad20589a1a2d86a8d693647fa8385408a4c17b0 \ + --hash=sha256:207aa914576b1181597a1516e1b90599dc690c095343ae281b0772e44945e6a4 \ + --hash=sha256:219a7db5ed53e523eb4a4769f13105118b6d5b04ed169a283c9775af221e231f \ + --hash=sha256:2b50ea79849e46b5f4f2b0247a3687505d32d161eeb16a75f6f7e6cd81936e43 \ + --hash=sha256:5903d38362b6c716e66bbe470f190579c530a5baf03dbc8500e5c2357aa569a5 \ + --hash=sha256:5c24903675e271bd688c6e9e7df5775ac6b168feb87dbe0e4bcc90805f21b28f \ + --hash=sha256:5ef6bc5ed98139e084f4e91100f2b098a0cd3493d4e76f9d6b3f7b95d7ad0f06 \ + --hash=sha256:61b55ae3c23a126a788b33ffb18f37d6668e79a05e756588d9e4d4be7246ab1c \ + --hash=sha256:63ddb992931a5e616c87d3d89f5a58db086e617548005c7f9059fac68c03a5cc \ + --hash=sha256:6943da9c09870490dcfd50c4909c0cc19f434fa6948f61282dc9cb07bcf08160 \ + --hash=sha256:6ad40f85c1207803d581d5d75e9ea25327cd524925699a83dfc03bf8e4ba72b7 \ + --hash=sha256:6b44433a79bdd7af0e3337bd7bbcf53dd1f9b0fa66bf21bcb756060ce32a96c1 \ + --hash=sha256:6bbaa245015d933a4172395baad7874373f162955d73612f0b66b6c2c33b6366 \ + --hash=sha256:7007227f4ea85b40a2f5e5a244479f6a6dfcf906db9b55e812a814a8f0e2c28d \ + --hash=sha256:74884a0aec1f1609190ec8b34b5d58fb3b5353cf22b96161e13e0e835f13518f \ + --hash=sha256:7d25fe5571ddb16369054f54cdd883f23de9941476d97f2b92eb6d7d83afe22d \ + --hash=sha256:7e162bdc5e3baad26b2262240be7d2bab36991d85a6a556e48b9dfb402370261 \ + --hash=sha256:814d62678dc3a30f4aa081982d830b7c342cf230ffc9d030b020cb154eeebf9e \ + --hash=sha256:8878a34c5313ee52e20aa50b03138af8d472bae465710fb954d133a9bfd3c38d \ + --hash=sha256:a66a0d94e5b081d5d695e66d6667e91e74d79e273eee95c1747717ba9cb70792 \ + --hash=sha256:a69f5cbf4addcfdf03dda564a671040127a6b7c34cf9fe4973582e68441b63fa \ + --hash=sha256:b00f9f0c334d07709d3f73a7cb8ae63c6ca1a90c790a63b5e7effa666ef96021 \ + --hash=sha256:b6ed71e4a7b4690447b626f499d978aa13197a0e592950e5d7020308f6054698 \ + --hash=sha256:bdf5041e5851526e885af579d2f455348dba68d74f14a32781933569a327fddf \ + --hash=sha256:be034360dd34e62608419f86e799c97d389c10a0e677a25f236a971b2f40dac9 \ + --hash=sha256:cc8f590a5eed30b314ae6b0232d925519ade433f663de79cc3783e4b10d662ba \ + --hash=sha256:cd7a318a15fe6cc4584bf3c4426f092ed08c0fd012cf2a9173114234fe193e11 \ + --hash=sha256:cf19b5f63a59c20306e034e691402b02055c8f4e38bf6792c23cad489162a642 \ + --hash=sha256:cfc781ce442ec407c841e9aa51d0e1024f72b6ec34caa8fdb6ef9576d549acf2 \ + --hash=sha256:dea9f6f8633571e18bc20cad83603072e697103a567f4b0738d52dd0211b4527 \ + --hash=sha256:e4a86a1d5eb2cce83c5972b3930c7c1eac81ab3508464345e2b8e54f119d5505 \ + --hash=sha256:e7106374d4a74ed9ff00c46cc00f0a9f06a0775f8868e423f85d4464d2333679 \ + --hash=sha256:e98a8a585b5668aa9e34d10f7785abf9545fe72663b4bfc16c99a115185ae6a5 \ + --hash=sha256:f64840e68483316eb58d82c376ad3585ca995e69e33b230436de0cdddf7363f9 \ + --hash=sha256:f8f4b0a9e6683e43889852130595c8854d8ae237f2324a053cdd884de936aa9b \ + --hash=sha256:fc45a53219ed30a7f670a6d8c98527af0020e6fd4ee4c0a8fb59f147f06d816c diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index f5140f9c7..4f5eda34e 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,7 +1,7 @@ # Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in letsencrypt-auto's requirements file. # Some dev package versions specified here may be overridden by higher level constraints -# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). +# files during tests (eg. tools/certbot_constraints.txt). alabaster==0.7.10 apacheconfig==0.3.2 apipkg==1.4 @@ -16,8 +16,8 @@ backports.functools-lru-cache==1.5 backports.shutil-get-terminal-size==1.0.0 backports.ssl-match-hostname==3.7.0.1 bcrypt==3.1.6 -boto3==1.11.7 -botocore==1.14.7 +boto3==1.17.4 +botocore==1.20.4 cached-property==1.5.1 cloudflare==2.3.1 configparser==3.7.4 @@ -51,11 +51,12 @@ jedi==0.17.1 Jinja2==2.9.6 jmespath==0.9.4 josepy==1.1.0 +jsonpickle==2.0.0 jsonschema==2.6.0 lazy-object-proxy==1.4.3 logger==1.4 logilab-common==1.4.1 -MarkupSafe==1.0 +MarkupSafe==1.1.1 mccabe==0.6.1 more-itertools==5.0.0 msrest==0.6.18 @@ -93,7 +94,7 @@ pytest-xdist==1.22.5 pytest-sugar==0.9.2 pytest-rerunfailures==4.2 python-dateutil==2.8.1 -python-digitalocean==1.11 +python-digitalocean==1.15.0 python-dotenv==0.14.0 pywin32==300 PyYAML==5.3.1 diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index 0d3626853..d2ebe3537 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -14,10 +14,6 @@ WORKDIR /opt/certbot # Copy certbot code COPY CHANGELOG.md README.rst src/ -# We keep the relative path to the requirements file the same because, as of -# writing this, tools/pip_install.py is used in the Dockerfile for Certbot -# plugins and this script expects to find the requirements file there. -COPY letsencrypt-auto-source/pieces/dependency-requirements.txt letsencrypt-auto-source/pieces/ COPY tools tools COPY acme src/acme COPY certbot src/certbot diff --git a/tools/extract_changelog.py b/tools/extract_changelog.py index fb0b849aa..bd718191a 100755 --- a/tools/extract_changelog.py +++ b/tools/extract_changelog.py @@ -24,7 +24,7 @@ def main(): i = 0 while i < len(lines): if section_pattern.match(lines[i]): - i = i + 1 + i = i + 2 while i < len(lines): if NEW_SECTION_PATTERN.match(lines[i]): break @@ -32,8 +32,6 @@ def main(): i = i + 1 i = i + 1 - changelog = [entry for entry in changelog if entry] - print('\n'.join(changelog)) diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 0b47fa5f8..e7a34286c 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -5,8 +5,6 @@ # set to 1, packages are installed using pinned versions of all of our # dependencies. See pip_install.py for more information on the versions pinned # to. -from __future__ import print_function - import os import re import subprocess diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index bbcb38051..44a61249b 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -6,8 +6,6 @@ Only the simple formats SomeProject==1.2.3 or SomeProject<=1.2.3 are currently supported. """ -from __future__ import print_function - import sys diff --git a/tools/pip_install.py b/tools/pip_install.py index c1c81482b..e06650ff2 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -57,7 +57,7 @@ def certbot_oldest_processing(tools_path, args, test_constraints): def certbot_normal_processing(tools_path, test_constraints): repo_path = os.path.dirname(tools_path) certbot_requirements = os.path.normpath(os.path.join( - repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) + repo_path, 'tools/certbot_constraints.txt')) with open(certbot_requirements, 'r') as fd: certbot_reqs = fd.readlines() with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd: @@ -76,8 +76,7 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain # Here is the order by increasing priority: # 1) The general development constraints (tools/dev_constraints.txt) # 2) The general tests constraints (oldest_requirements.txt or - # certbot-auto's dependency-requirements.txt + pipstrap's constraints - # for the normal processing) + # certbot_constraints.txt + pipstrap's constraints for the normal processing) # 3) The local requirement file, typically local-oldest-requirement in oldest tests files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] if requirements: @@ -134,6 +133,7 @@ def main(args): pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"' .format(requirements)) + print(' '.join(args)) pip_install_with_print(' '.join(args), env=env) diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index abfe9f214..de2a0ff57 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -6,9 +6,6 @@ # https://github.com/pyca/cryptography/blob/a02fdd60d98273ca34427235c4ca96687a12b239/.travis/downstream.d/certbot.sh#L8-L9. # We should try to remember to keep their repo updated if we make any changes # to this script which may break things for them. - -from __future__ import absolute_import - import sys import pip_install diff --git a/tools/pipstrap.py b/tools/pipstrap.py index e6b746916..2b2e3dcbb 100755 --- a/tools/pipstrap.py +++ b/tools/pipstrap.py @@ -1,11 +1,9 @@ #!/usr/bin/env python """Uses pip to upgrade Python packaging tools to pinned versions.""" -from __future__ import absolute_import import os import pip_install - _REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_constraints.txt") diff --git a/tools/pipstrap_constraints.txt b/tools/pipstrap_constraints.txt index 5de9e147d..54ab8b429 100644 --- a/tools/pipstrap_constraints.txt +++ b/tools/pipstrap_constraints.txt @@ -10,9 +10,9 @@ pip==20.2.4 \ --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \ --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1 -setuptools==44.1.1 \ - --hash=sha256:27a714c09253134e60a6fa68130f78c7037e5562c4f21f8f318f2ae900d152d5 \ - --hash=sha256:c67aa55db532a0dadc4d2e20ba9961cbd3ccc84d544e9029699822542b5a476b +setuptools==54.1.2 \ + --hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b \ + --hash=sha256:ebd0148faf627b569c8d2a1b20f5d3b09c873f12739d71c7ee88f037d5be82ff wheel==0.35.1 \ --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \ --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f diff --git a/tools/readlink.py b/tools/readlink.py index 446c8ebdc..c1e0c2e61 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -6,8 +6,6 @@ useful as there are often differences in readlink on different platforms. """ -from __future__ import print_function - import os import sys diff --git a/letsencrypt-auto-source/rebuild_dependencies.py b/tools/rebuild_certbot_constraints.py similarity index 84% rename from letsencrypt-auto-source/rebuild_dependencies.py rename to tools/rebuild_certbot_constraints.py index 864394661..f5e5d3ca7 100755 --- a/letsencrypt-auto-source/rebuild_dependencies.py +++ b/tools/rebuild_certbot_constraints.py @@ -4,12 +4,12 @@ Gather and consolidate the up-to-date dependencies available and required to ins on various Linux distributions. It generates a requirements file contained the pinned and hashed versions, ready to be used by pip to install the certbot dependencies. -This script is typically used to update the certbot-requirements.txt file of certbot-auto. +This script is typically used to update the certbot_constraints.txt file. To achieve its purpose, this script will start a certbot installation with unpinned dependencies, then gather them, on various distributions started as Docker containers. -Usage: letsencrypt-auto-source/rebuild_dependencies new_requirements.txt +Usage: tools/rebuild_certbot_constraints.py new_requirements.txt NB1: Docker must be installed on the machine running this script. NB2: Python library 'hashin' must be installed on the machine running this script. @@ -26,52 +26,41 @@ import argparse # The list of docker distributions to test dependencies against with. DISTRIBUTION_LIST = [ - 'ubuntu:18.04', 'ubuntu:16.04', - 'debian:stretch', - 'centos:7', 'centos:6', - 'opensuse/leap:15', - 'fedora:29', + 'ubuntu:20.04', 'ubuntu:18.04', 'debian:buster', + 'centos:8', 'centos:7', 'fedora:29', ] # These constraints will be added while gathering dependencies on each distribution. # It can be used because a particular version for a package is required for any reason, # or to solve a version conflict between two distributions requirements. AUTHORITATIVE_CONSTRAINTS = { - # Using an older version of mock here prevents regressions of #5276. - 'mock': '1.3.0', # Too touchy to move to a new version. And will be removed soon # in favor of pure python parser for Apache. 'python-augeas': '0.5.0', - # Package enum34 needs to be explicitly limited to Python2.x, in order to avoid - # certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456. - 'enum34': '1.1.10; python_version < \'3.4\'', - # Cryptography 2.9+ drops support for OpenSSL 1.0.1, but we still want to support it - # for officially supported non-x86_64 ancient distributions like RHEL 6. - 'cryptography': '2.8', - # Parsedatetime 2.6 is broken on Python 2.7, see https://github.com/bear/parsedatetime/issues/246 - 'parsedatetime': '2.5', + # We avoid cryptography 3.4+ since it requires Rust to compile the wheels, and + # this needs some work on the snap builds. + 'cryptography': '3.3.2', } -# ./certbot/letsencrypt-auto-source/rebuild_dependencies.py (2 levels from certbot root path) +# ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path) CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__))) # The script will be used to gather dependencies for a given distribution. -# - certbot-auto is used to install relevant OS packages, and set up an initial venv +# - bootstrap_os_packages.sh is used to install relevant OS packages, and set up an initial venv # - then this venv is used to consistently construct an empty new venv -# - once pipstraped, this new venv pip-installs certbot runtime (including apache/nginx), +# - once pipstrap.py, this new venv pip-installs certbot runtime (including apache/nginx), # without pinned dependencies, and respecting input authoritative requirements # - `certbot plugins` is called to check we have a healthy environment # - finally current set of dependencies is extracted out of the docker using pip freeze SCRIPT = r"""#!/bin/sh -set -e +set -ex cd /tmp/certbot -letsencrypt-auto-source/letsencrypt-auto --install-only -n -PYVER=`/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` +tests/letstest/scripts/bootstrap_os_packages.sh -/opt/eff.org/certbot/venv/bin/python letsencrypt-auto-source/pieces/create_venv.py /tmp/venv "$PYVER" 1 +python3 -m venv /tmp/venv -/tmp/venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py +/tmp/venv/bin/python tools/pipstrap.py /tmp/venv/bin/pip install -e acme -e certbot -e certbot-apache -e certbot-nginx -c /tmp/constraints.txt /tmp/venv/bin/certbot plugins /tmp/venv/bin/pip freeze >> /tmp/workspace/requirements.txt @@ -109,6 +98,7 @@ def _requirements_from_one_distribution(distribution, verbose): '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) command = ['docker', 'run', '--rm', '--cidfile', cid_file, + '--network=host', '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), '-v', '{0}:/tmp/workspace'.format(workspace), '-v', '{0}:/tmp/constraints.txt'.format(authoritative_constraints), @@ -158,7 +148,7 @@ def _parse_and_merge_requirements(dependencies_map, requirements_file_lines, dis """ for line in requirements_file_lines: match = re.match(r'([^=]+)==([^=]+)', line.strip()) - if not line.startswith('-e') and match: + if not line.startswith('-e') and not line.startswith('#') and match: package, version = match.groups() if package not in ['acme', 'certbot', 'certbot-apache', 'certbot-nginx', 'pkg-resources']: dependencies_map.setdefault(package, []).append((version, distribution)) @@ -215,11 +205,11 @@ def _write_requirements(dest_file, requirements, conflicts): print('===> Calculating hashes for the requirement file.') _write_to(dest_file, '''\ -# This is the flattened list of packages certbot-auto installs. +# This is the flattened list of pinned packages to build certbot deployable artifacts. # To generate this, do (with docker and package hashin installed): # ``` -# letsencrypt-auto-source/rebuild_dependencies.py \\ -# letsencrypt-auto-source/pieces/dependency-requirements.txt +# tools/rebuild_certbot_contraints.py \\ +# tools/certbot_constraints.txt # ``` # If you want to update a single dependency, run commands similar to these: # ``` @@ -264,8 +254,8 @@ def _gather_dependencies(dest_file, verbose): if __name__ == '__main__': parser = argparse.ArgumentParser( - description=('Build a sanitized, pinned and hashed requirements file for certbot-auto, ' - 'validated against several OS distributions using Docker.')) + description=('Build a sanitized, pinned and hashed requirements file for certbot deployable' + ' artifacts, validated against several OS distributions using Docker.')) parser.add_argument('requirements_path', help='path for the generated requirements file') parser.add_argument('--verbose', '-v', action='store_true', diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index e6a44240f..69c704e45 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -2,32 +2,59 @@ import argparse import datetime import glob +from multiprocessing import Manager +from multiprocessing import Pool +from multiprocessing import Process +from multiprocessing.managers import SyncManager +import os +from os.path import basename +from os.path import dirname +from os.path import exists +from os.path import join +from os.path import realpath import re import subprocess import sys +import tempfile +from threading import Lock import time -from multiprocessing import Pool, Process, Manager -from os.path import join, realpath, dirname, basename, exists +from typing import Dict +from typing import List +from typing import Set +from typing import Tuple CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] -def _execute_build(target, archs, status, workspace): - process = subprocess.Popen([ - 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', - '--build-on', ','.join(archs) - ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, cwd=workspace) +def _execute_build( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + workspace: str) -> Tuple[int, List[str]]: - process_output = [] + with tempfile.TemporaryDirectory() as tempdir: + environ = os.environ.copy() + environ['XDG_CACHE_HOME'] = tempdir + process = subprocess.Popen([ + 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', + '--build-on', ','.join(archs)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True, env=environ, cwd=workspace) + + process_output: List[str] = [] for line in process.stdout: process_output.append(line) _extract_state(target, line, status) + if any(state for state in status[target].values() if state == 'Chroot problem'): + # On this error the snapcraft process stales. Let's finish it. + process.kill() + return process.wait(), process_output -def _build_snap(target, archs, status, running, lock): +def _build_snap( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + running: Dict[str, bool], lock: Lock) -> Dict[str, str]: status[target] = {arch: '...' for arch in archs} if target == 'certbot': @@ -45,7 +72,14 @@ def _build_snap(target, archs, status, running, lock): with lock: dump_output = exit_code != 0 - failed_archs = [arch for arch in archs if status[target][arch] == 'Failed to build'] + failed_archs = [arch for arch in archs if status[target][arch] != 'Successfully built'] + if any(arch for arch in archs if status[target][arch] == 'Chroot problem'): + print('Some builds failed with the status "Chroot problem".') + print('This status is known to make any future build fail until either ' + 'the source code changes or the build on Launchpad is deleted.') + print('Please fix the build appropriately before trying a new one.') + # It is useless to retry in this situation. + retry = 0 if exit_code == 0 and not failed_archs: # We expect to have all target snaps available, or something bad happened. snaps_list = glob.glob(join(workspace, '*.snap')) @@ -74,7 +108,7 @@ def _build_snap(target, archs, status, running, lock): return {target: workspace} -def _extract_state(project, output, status): +def _extract_state(project: str, output: str, status: Dict[str, Dict[str, str]]) -> None: match = re.match(r'^.*arch=(\w+)\s+state=([\w ]+).*$', output) if match: arch = match.group(1) @@ -89,7 +123,7 @@ def _extract_state(project, output, status): status[project] = state -def _dump_status_helper(archs, status): +def _dump_status_helper(archs: Set[str], status: Dict[str, Dict[str, str]]) -> None: headers = ['project', *archs] print(''.join(f'| {item:<25}' for item in headers)) print(f'|{"-" * 26}' * len(headers)) @@ -101,14 +135,18 @@ def _dump_status_helper(archs, status): sys.stdout.flush() -def _dump_status(archs, status, running): +def _dump_status( + archs: Set[str], status: Dict[str, Dict[str, str]], + running: Dict[str, bool]) -> None: while any(running.values()): print(f'Remote build status at {datetime.datetime.now()}') _dump_status_helper(archs, status) time.sleep(10) -def _dump_results(targets, archs, status, workspaces): +def _dump_results( + targets: Set[str], archs: Set[str], status: Dict[str, Dict[str, str]], + workspaces: Dict[str, str]) -> bool: failures = False for target in targets: for arch in archs: @@ -146,8 +184,8 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('targets', nargs='+', choices=['ALL', 'DNS_PLUGINS', 'certbot', *PLUGINS], help='the list of snaps to build') - parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default=['amd64'], - help='the architectures for which snaps are built') + parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], + default=['amd64'], help='the architectures for which snaps are built') parser.add_argument('--timeout', type=int, default=None, help='build process will fail after the provided timeout (in seconds)') args = parser.parse_args() @@ -174,8 +212,10 @@ def main(): print(f' - projects: {", ".join(sorted(targets))}') print() - with Manager() as manager, Pool(processes=len(targets)) as pool: - status = manager.dict() + manager: SyncManager = Manager() + pool = Pool(processes=len(targets)) + with manager, pool: + status: Dict[str, Dict[str, str]] = manager.dict() running = manager.dict({target: True for target in targets}) lock = manager.Lock() diff --git a/tools/snap/generate_dnsplugins_all.sh b/tools/snap/generate_dnsplugins_all.sh index 40404bf9b..976b0dd7b 100755 --- a/tools/snap/generate_dnsplugins_all.sh +++ b/tools/snap/generate_dnsplugins_all.sh @@ -10,7 +10,7 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH # Create constraints file "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt) \ + <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_constraints.txt) \ <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \ > "${PLUGIN_PATH}"/snap-constraints.txt done diff --git a/windows-installer/construct.py b/windows-installer/construct.py index 0684b3c25..eb199a7e1 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -2,6 +2,7 @@ import contextlib import ctypes import os +import re import shutil import struct import subprocess @@ -9,7 +10,7 @@ import sys import tempfile import time -PYTHON_VERSION = (3, 8, 6) +PYTHON_VERSION = (3, 8, 8) PYTHON_BITNESS = 32 PYWIN32_VERSION = 300 # do not forget to edit pywin32 dependency accordingly in setup.py NSIS_VERSION = '3.06.1' @@ -52,6 +53,21 @@ def _compile_wheels(repo_path, build_path, venv_python): command.extend(wheels_project) subprocess.check_call(command, env=env) + # Cryptography uses now a unique wheel name "cryptography-VERSION-cpXX-abi3-win32.whl where + # cpXX is the lowest supported version of Python (eg. cp36 says that the wheel is compatible + # with Python 3.6+). While technically valid to describe a wheel compliant with the Stable + # Application Binary Interface, this naming convention makes pynsist falsely think that the + # wheel is compatible with Python 3.6 only. + # Let's trick pynsist by renaming the wheel until this is fixed upstream. + for file in os.listdir(wheels_path): + # Given that our Python version is 3.8, this rename files like + # cryptography-VERSION-cpXX-abi3-win32.whl into cryptography-VERSION-cp38-abi3-win32.whl + renamed = re.sub(r'^(.*)-cp\d+-abi3-(\w+)\.whl$', r'\1-cp{0}{1}-abi3-\2.whl' + .format(PYTHON_VERSION[0], PYTHON_VERSION[1]), file) + print(renamed) + if renamed != file: + os.replace(os.path.join(wheels_path, file), os.path.join(wheels_path, renamed)) + def _prepare_build_tools(venv_path, venv_python, repo_path): print('Prepare build tools') @@ -63,7 +79,7 @@ def _prepare_build_tools(venv_path, venv_python, repo_path): @contextlib.contextmanager def _prepare_constraints(repo_path): - reqs_certbot = os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'dependency-requirements.txt') + reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_constraints.txt') reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt') constraints_certbot = subprocess.check_output( [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot],