diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml index cffedfcb2..c8cbe7fb3 100644 --- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml @@ -85,12 +85,6 @@ jobs: IMAGE_NAME: macOS-10.15 PYTHON_VERSION: 3.8 TOXENV: test-farm-apache2 - farmtest-leauto-upgrades: - PYTHON_VERSION: 3.7 - TOXENV: test-farm-leauto-upgrades - farmtest-certonly-standalone: - PYTHON_VERSION: 3.7 - TOXENV: test-farm-certonly-standalone farmtest-sdists: PYTHON_VERSION: 3.7 TOXENV: test-farm-sdists diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index 28255919f..005b5ef48 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -12,6 +12,9 @@ jobs: DOCKER_ARCH: arm32v6 arm64v8: DOCKER_ARCH: arm64v8 + # The default timeout of 60 minutes is a little low for compiling + # cryptography on ARM architectures. + timeoutInMinutes: 180 steps: - bash: set -e && tools/docker/build.sh $(dockerTag) $DOCKER_ARCH displayName: Build the Docker images @@ -59,7 +62,13 @@ jobs: versionSpec: 3.8 architecture: x86 addToPath: true - - script: python windows-installer/construct.py + - script: | + python -m venv venv + venv\Scripts\python tools\pipstrap.py + venv\Scripts\python tools\pip_install.py -e windows-installer + displayName: Prepare Windows installer build environment + - script: | + venv\Scripts\construct-windows-installer displayName: Build Certbot installer - task: CopyFiles@2 inputs: @@ -116,13 +125,17 @@ jobs: - job: snaps_build pool: vmImage: ubuntu-18.04 + strategy: + matrix: + amd64: + SNAP_ARCH: amd64 + # Do not run the heavy non-amd64 builds for test branches + ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}: + armhf: + SNAP_ARCH: armhf + arm64: + SNAP_ARCH: arm64 timeoutInMinutes: 0 - variables: - # Do not run the heavy non-amd64 builds for test branches - ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}: - ARCHS: amd64 arm64 armhf - ${{ if startsWith(variables['Build.SourceBranchName'], 'test-') }}: - ARCHS: amd64 steps: - script: | set -e @@ -144,7 +157,7 @@ jobs: git config --global user.name "$(Build.RequestedFor)" mkdir -p ~/.local/share/snapcraft/provider/launchpad cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials - python3 tools/snap/build_remote.py ALL --archs ${ARCHS} --timeout 19800 + python3 tools/snap/build_remote.py ALL --archs ${SNAP_ARCH} --timeout 19800 displayName: Build snaps - script: | set -e @@ -154,7 +167,7 @@ jobs: - task: PublishPipelineArtifact@1 inputs: path: $(Build.ArtifactStagingDirectory) - artifact: snaps + artifact: snaps_$(SNAP_ARCH) displayName: Store snaps artifacts - job: snap_run dependsOn: snaps_build @@ -175,12 +188,12 @@ jobs: displayName: Install dependencies - task: DownloadPipelineArtifact@2 inputs: - artifact: snaps + artifact: snaps_amd64 path: $(Build.SourcesDirectory)/snap displayName: Retrieve Certbot snaps - script: | set -e - sudo snap install --dangerous --classic snap/certbot_*_amd64.snap + sudo snap install --dangerous --classic snap/certbot_*.snap displayName: Install Certbot snap - script: | set -e @@ -202,7 +215,7 @@ jobs: addToPath: true - task: DownloadPipelineArtifact@2 inputs: - artifact: snaps + artifact: snaps_amd64 path: $(Build.SourcesDirectory)/snap displayName: Retrieve Certbot snaps - script: | diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index fec11c6c5..c949af44a 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -40,13 +40,13 @@ jobs: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.9 TOXENV: py39-cover - linux-py37-lint: + linux-py39-lint: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 3.7 + PYTHON_VERSION: 3.9 TOXENV: lint - linux-py36-mypy: + linux-py39-mypy: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 3.6 + PYTHON_VERSION: 3.9 TOXENV: mypy linux-integration: IMAGE_NAME: ubuntu-18.04 @@ -56,6 +56,8 @@ jobs: apache-compat: IMAGE_NAME: ubuntu-18.04 TOXENV: apache_compat + # le-modification can be moved to the extended test suite once + # https://github.com/certbot/certbot/issues/8742 is resolved. le-modification: IMAGE_NAME: ubuntu-18.04 TOXENV: modification diff --git a/.azure-pipelines/templates/stages/deploy-stage.yml b/.azure-pipelines/templates/stages/deploy-stage.yml index ac2044f99..abbb1fd1a 100644 --- a/.azure-pipelines/templates/stages/deploy-stage.yml +++ b/.azure-pipelines/templates/stages/deploy-stage.yml @@ -37,6 +37,14 @@ stages: vmImage: ubuntu-18.04 variables: - group: certbot-common + strategy: + matrix: + amd64: + SNAP_ARCH: amd64 + arm32v6: + SNAP_ARCH: armhf + arm64v8: + SNAP_ARCH: arm64 steps: - bash: | set -e @@ -46,7 +54,7 @@ stages: displayName: Install dependencies - task: DownloadPipelineArtifact@2 inputs: - artifact: snaps + artifact: snaps_$(SNAP_ARCH) path: $(Build.SourcesDirectory)/snap displayName: Retrieve Certbot snaps - task: DownloadSecureFile@1 @@ -55,8 +63,7 @@ stages: secureFile: snapcraft.cfg - bash: | set -e - mkdir -p .snapcraft - ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg + snapcraft login --with $(snapcraftCfg.secureFilePath) for SNAP_FILE in snap/*.snap; do tools/retry.sh eval snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}" done diff --git a/.azure-pipelines/templates/steps/sphinx-steps.yml b/.azure-pipelines/templates/steps/sphinx-steps.yml index 23c258bbc..7e1d2ee9d 100644 --- a/.azure-pipelines/templates/steps/sphinx-steps.yml +++ b/.azure-pipelines/templates/steps/sphinx-steps.yml @@ -9,7 +9,7 @@ steps: do echo "" echo "##[group]Building $doc_path" - pip install -q -e $doc_path/..[docs] + tools/pip_install_editable.py $doc_path/..[docs] if ! sphinx-build -W --keep-going -b html $doc_path $doc_path/_build/html; then FINAL_STATUS=1 FAILED_BUILDS[${#FAILED_BUILDS[@]}]="${doc_path%/docs}" diff --git a/.azure-pipelines/templates/steps/tox-steps.yml b/.azure-pipelines/templates/steps/tox-steps.yml index ecf3d6032..3e5fb995d 100644 --- a/.azure-pipelines/templates/steps/tox-steps.yml +++ b/.azure-pipelines/templates/steps/tox-steps.yml @@ -1,6 +1,10 @@ steps: + # We run brew update because we've seen attempts to install an older version + # of a package fail. See + # https://github.com/actions/virtual-environments/issues/3165. - bash: | set -e + brew update brew install augeas condition: startswith(variables['IMAGE_NAME'], 'macOS') displayName: Install MacOS dependencies diff --git a/.gitignore b/.gitignore index 5169defd6..285e68a42 100644 --- a/.gitignore +++ b/.gitignore @@ -4,13 +4,12 @@ build/ dist*/ /venv*/ -/kgs/ /.tox/ /releases*/ /log* letsencrypt.log certbot.log -letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64 +poetry.lock # coverage .coverage @@ -31,12 +30,6 @@ tags # auth --cert-path --chain-path /*.pem -# letstest -tests/letstest/letest-*/ -tests/letstest/*.pem -tests/letstest/venv/ -tests/letstest/venv3/ - .venv # pytest cache diff --git a/.isort.cfg b/.isort.cfg index 11c895f4d..6b17b459b 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,5 @@ [settings] skip_glob=venv* -skip=letsencrypt-auto-source force_sort_within_sections=True force_single_line=True order_by_type=False diff --git a/.pylintrc b/.pylintrc index a2468b0cf..e19077d8d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -8,7 +8,10 @@ jobs=0 # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). -#init-hook= +# CERTBOT COMMENT +# This is needed for pylint to import linter_plugin.py since +# https://github.com/PyCQA/pylint/pull/3396. +init-hook="import pylint.config, os, sys; sys.path.append(os.path.dirname(pylint.config.PYLINTRC))" # Profiled execution. profile=no diff --git a/Dockerfile-dev b/Dockerfile-dev index 86847f8fd..895dbdc0b 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,5 +1,5 @@ # This Dockerfile builds an image for development. -FROM debian:buster +FROM ubuntu:focal # Note: this only exposes the port to other docker containers. EXPOSE 80 443 @@ -8,8 +8,9 @@ WORKDIR /opt/certbot/src COPY . . RUN apt-get update && \ - apt-get install apache2 git python3-dev python3-venv gcc libaugeas0 \ - libssl-dev libffi-dev ca-certificates openssl nginx-light -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install apache2 git python3-dev \ + python3-venv gcc libaugeas0 libssl-dev libffi-dev ca-certificates \ + openssl nginx-light -y --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 03917f8ca..87ecddc6c 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -7,7 +7,7 @@ questions. ## My operating system is (include version): -## I installed Certbot with (certbot-auto, OS package manager, pip, etc): +## I installed Certbot with (snap, OS package manager, pip, certbot-auto, etc): ## I ran this command and it produced this output: diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 41a2aa258..2c8190be5 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -5,17 +5,19 @@ import functools import hashlib import logging import socket +from typing import Type from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose -import requests -from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from OpenSSL import crypto +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 +import requests from acme import crypto_util from acme import errors from acme import fields -from acme.mixins import ResourceMixin, TypeMixin +from acme.mixins import ResourceMixin +from acme.mixins import TypeMixin logger = logging.getLogger(__name__) @@ -23,12 +25,12 @@ logger = logging.getLogger(__name__) class Challenge(jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge.""" - TYPES = {} # type: dict + TYPES: dict = {} @classmethod def from_json(cls, jobj): try: - return super(Challenge, cls).from_json(jobj) + return super().from_json(jobj) except jose.UnrecognizedTypeError as error: logger.debug(error) return UnrecognizedChallenge.from_json(jobj) @@ -37,7 +39,7 @@ class Challenge(jose.TypedJSONObjectWithFields): class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge response.""" - TYPES = {} # type: dict + TYPES: dict = {} resource_type = 'challenge' resource = fields.Resource(resource_type) @@ -56,7 +58,7 @@ class UnrecognizedChallenge(Challenge): """ def __init__(self, jobj): - super(UnrecognizedChallenge, self).__init__() + super().__init__() object.__setattr__(self, "jobj", jobj) def to_partial_json(self): @@ -139,7 +141,7 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return True def to_partial_json(self): - jobj = super(KeyAuthorizationChallengeResponse, self).to_partial_json() + jobj = super().to_partial_json() jobj.pop('keyAuthorization', None) return jobj @@ -151,8 +153,8 @@ class KeyAuthorizationChallenge(_TokenChallenge, metaclass=abc.ABCMeta): that will be used to generate ``response``. :param str typ: type of the challenge """ - typ = NotImplemented - response_cls = NotImplemented + typ: str = NotImplemented + response_cls: Type[KeyAuthorizationChallengeResponse] = NotImplemented thumbprint_hash_function = ( KeyAuthorizationChallengeResponse.thumbprint_hash_function) diff --git a/acme/acme/client.py b/acme/acme/client.py index c3f8c550f..548c3d548 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -8,6 +8,12 @@ import http.client as http_client import logging import re import time +from typing import cast +from typing import Dict +from typing import List +from typing import Set +from typing import Text +from typing import Union import josepy as jose import OpenSSL @@ -20,10 +26,6 @@ from acme import crypto_util from acme import errors from acme import jws from acme import messages -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Text from acme.mixins import VersionedLEACMEMixin logger = logging.getLogger(__name__) @@ -33,7 +35,7 @@ DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class ClientBase(object): +class ClientBase: """ACME client base object. :ivar messages.Directory directory: @@ -112,8 +114,9 @@ class ClientBase(object): """ return self.update_registration(regr, update={'status': 'deactivated'}) - def deactivate_authorization(self, authzr): - # type: (messages.AuthorizationResource) -> messages.AuthorizationResource + def deactivate_authorization(self, + authzr: messages.AuthorizationResource + ) -> messages.AuthorizationResource: """Deactivate authorization. :param messages.AuthorizationResource authzr: The Authorization resource @@ -250,7 +253,7 @@ class Client(ClientBase): if isinstance(directory, str): directory = messages.Directory.from_json( net.get(directory).json()) - super(Client, self).__init__(directory=directory, + super().__init__(directory=directory, net=net, acme_version=1) def register(self, new_reg=None): @@ -423,7 +426,7 @@ class Client(ClientBase): """ assert max_attempts > 0 - attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int] + attempts: Dict[messages.AuthorizationResource, int] = collections.defaultdict(int) exhausted = set() # priority queue with datetime.datetime (based on Retry-After) as key, @@ -536,7 +539,7 @@ class Client(ClientBase): :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509` """ - chain = [] # type: List[jose.ComparableX509] + chain: List[jose.ComparableX509] = [] uri = certr.cert_chain_uri while uri is not None and len(chain) < max_length: response, cert = self._get_cert(uri) @@ -574,7 +577,7 @@ class ClientV2(ClientBase): :param .messages.Directory directory: Directory Resource :param .ClientNetwork net: Client network. """ - super(ClientV2, self).__init__(directory=directory, + super().__init__(directory=directory, net=net, acme_version=2) def new_account(self, new_account): @@ -624,7 +627,7 @@ class ClientV2(ClientBase): """ # https://github.com/certbot/certbot/issues/6155 new_regr = self._get_v2_account(regr) - return super(ClientV2, self).update_registration(new_regr, update) + return super().update_registration(new_regr, update) def _get_v2_account(self, regr): self.net.account = None @@ -795,7 +798,7 @@ class ClientV2(ClientBase): if 'rel' in l and 'url' in l and l['rel'] == relation_type] -class BackwardsCompatibleClientV2(object): +class BackwardsCompatibleClientV2: """ACME client wrapper that tends towards V2-style calls, but supports V1 servers. @@ -817,6 +820,7 @@ class BackwardsCompatibleClientV2(object): def __init__(self, net, key, server): directory = messages.Directory.from_json(net.get(server).json()) self.acme_version = self._acme_version_from_directory(directory) + self.client: Union[Client, ClientV2] if self.acme_version == 1: self.client = Client(directory, key=key, net=net) else: @@ -836,16 +840,18 @@ class BackwardsCompatibleClientV2(object): if check_tos_cb is not None: check_tos_cb(tos) if self.acme_version == 1: - regr = self.client.register(regr) + client_v1 = cast(Client, self.client) + regr = client_v1.register(regr) if regr.terms_of_service is not None: _assess_tos(regr.terms_of_service) - return self.client.agree_to_tos(regr) + return client_v1.agree_to_tos(regr) return regr else: - if "terms_of_service" in self.client.directory.meta: - _assess_tos(self.client.directory.meta.terms_of_service) + client_v2 = cast(ClientV2, self.client) + if "terms_of_service" in client_v2.directory.meta: + _assess_tos(client_v2.directory.meta.terms_of_service) regr = regr.update(terms_of_service_agreed=True) - return self.client.new_account(regr) + return client_v2.new_account(regr) def new_order(self, csr_pem): """Request a new Order object from the server. @@ -863,14 +869,15 @@ class BackwardsCompatibleClientV2(object): """ if self.acme_version == 1: + client_v1 = cast(Client, self.client) csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) # pylint: disable=protected-access dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) authorizations = [] for domain in dnsNames: - authorizations.append(self.client.request_domain_challenges(domain)) + authorizations.append(client_v1.request_domain_challenges(domain)) return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem) - return self.client.new_order(csr_pem) + return cast(ClientV2, self.client).new_order(csr_pem) def finalize_order(self, orderr, deadline, fetch_alternative_chains=False): """Finalize an order and obtain a certificate. @@ -885,8 +892,9 @@ class BackwardsCompatibleClientV2(object): """ if self.acme_version == 1: + client_v1 = cast(Client, self.client) csr_pem = orderr.csr_pem - certr = self.client.request_issuance( + certr = client_v1.request_issuance( jose.ComparableX509( OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)), orderr.authorizations) @@ -894,7 +902,7 @@ class BackwardsCompatibleClientV2(object): chain = None while datetime.datetime.now() < deadline: try: - chain = self.client.fetch_chain(certr) + chain = client_v1.fetch_chain(certr) break except errors.Error: time.sleep(1) @@ -909,7 +917,8 @@ class BackwardsCompatibleClientV2(object): chain = crypto_util.dump_pyopenssl_chain(chain).decode() return orderr.update(fullchain_pem=(cert + chain)) - return self.client.finalize_order(orderr, deadline, fetch_alternative_chains) + return cast(ClientV2, self.client).finalize_order( + orderr, deadline, fetch_alternative_chains) def revoke(self, cert, rsn): """Revoke certificate. @@ -935,10 +944,10 @@ class BackwardsCompatibleClientV2(object): Always return False for ACMEv1 servers, as it doesn't use External Account Binding.""" if self.acme_version == 1: return False - return self.client.external_account_required() + return cast(ClientV2, self.client).external_account_required() -class ClientNetwork(object): +class ClientNetwork: """Wrapper around requests that signs POSTs for authentication. Also adds user agent, and handles Content-Type. @@ -968,7 +977,7 @@ class ClientNetwork(object): self.account = account self.alg = alg self.verify_ssl = verify_ssl - self._nonces = set() # type: Set[Text] + self._nonces: Set[Text] = set() self.user_agent = user_agent self.session = requests.Session() self._default_timeout = timeout @@ -1128,6 +1137,7 @@ class ClientNetwork(object): # If content is DER, log the base64 of it instead of raw bytes, to keep # binary data out of the logs. + debug_content: Union[bytes, str] if response.headers.get("Content-Type") == DER_CONTENT_TYPE: debug_content = base64.b64encode(response.content) else: diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 4b58db328..749478bf5 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -5,15 +5,15 @@ import logging import os import re import socket +from typing import Callable +from typing import Tuple +from typing import Union import josepy as jose from OpenSSL import crypto from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from acme import errors -from acme.magic_typing import Callable -from acme.magic_typing import Tuple -from acme.magic_typing import Union logger = logging.getLogger(__name__) @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore -class _DefaultCertSelection(object): +class _DefaultCertSelection: def __init__(self, certs): self.certs = certs @@ -36,7 +36,7 @@ class _DefaultCertSelection(object): return self.certs.get(server_name, None) -class SSLSocket(object): # pylint: disable=too-few-public-methods +class SSLSocket: # pylint: disable=too-few-public-methods """SSL wrapper for sockets. :ivar socket sock: Original wrapped socket. @@ -93,7 +93,7 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) - class FakeConnection(object): + class FakeConnection: """Fake OpenSSL.SSL.Connection.""" # pylint: disable=missing-function-docstring @@ -168,7 +168,7 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu source_address[1] ) if any(source_address) else "" ) - socket_tuple = (host, port) # type: Tuple[str, int] + socket_tuple: Tuple[str, int] = (host, port) sock = socket.create_connection(socket_tuple, **socket_kwargs) # type: ignore except socket.error as error: raise errors.Error(error) @@ -256,7 +256,7 @@ def _pyopenssl_cert_or_req_san(cert_or_req): if isinstance(cert_or_req, crypto.X509): # pylint: disable=line-too-long - func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] + func: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] = crypto.dump_certificate else: func = crypto.dump_certificate_request text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 5ca5a4fa2..b47ed88da 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -28,13 +28,8 @@ class NonceError(ClientError): class BadNonce(NonceError): """Bad nonce error.""" - def __init__(self, nonce, error, *args, **kwargs): - # MyPy complains here that there is too many arguments for BaseException constructor. - # This is an error fixed in typeshed, see https://github.com/python/mypy/issues/4183 - # The fix is included in MyPy>=0.740, but upgrading it would bring dozen of errors due to - # new types definitions. So we ignore the error until the code base is fixed to match - # with MyPy>=0.740 referential. - super(BadNonce, self).__init__(*args, **kwargs) # type: ignore + def __init__(self, nonce, error, *args): + super().__init__(*args) self.nonce = nonce self.error = error @@ -52,9 +47,8 @@ class MissingNonce(NonceError): :ivar requests.Response ~.response: HTTP Response """ - def __init__(self, response, *args, **kwargs): - # See comment in BadNonce constructor above for an explanation of type: ignore here. - super(MissingNonce, self).__init__(*args, **kwargs) # type: ignore + def __init__(self, response, *args): + super().__init__(*args) self.response = response def __str__(self): @@ -78,7 +72,7 @@ class PollError(ClientError): def __init__(self, exhausted, updated): self.exhausted = exhausted self.updated = updated - super(PollError, self).__init__() + super().__init__() @property def timeout(self): @@ -96,7 +90,7 @@ class ValidationError(Error): """ def __init__(self, failed_authzrs): self.failed_authzrs = failed_authzrs - super(ValidationError, self).__init__() + super().__init__() class TimeoutError(Error): # pylint: disable=redefined-builtin @@ -112,7 +106,7 @@ class IssuanceError(Error): :param messages.Error error: The error provided by the server. """ self.error = error - super(IssuanceError, self).__init__() + super().__init__() class ConflictError(ClientError): @@ -125,7 +119,7 @@ class ConflictError(ClientError): """ def __init__(self, location): self.location = location - super(ConflictError, self).__init__() + super().__init__() class WildcardUnsupportedError(Error): diff --git a/acme/acme/fields.py b/acme/acme/fields.py index 3b5672283..bd915e47d 100644 --- a/acme/acme/fields.py +++ b/acme/acme/fields.py @@ -12,7 +12,7 @@ class Fixed(jose.Field): def __init__(self, json_name, value): self.value = value - super(Fixed, self).__init__( + super().__init__( json_name=json_name, default=value, omitempty=False) def decode(self, value): @@ -53,7 +53,7 @@ class Resource(jose.Field): def __init__(self, resource_type, *args, **kwargs): self.resource_type = resource_type - super(Resource, self).__init__( + super().__init__( 'resource', default=resource_type, *args, **kwargs) def decode(self, value): diff --git a/acme/acme/jws.py b/acme/acme/jws.py index 2188c3727..d9d3e12c3 100644 --- a/acme/acme/jws.py +++ b/acme/acme/jws.py @@ -14,7 +14,9 @@ class Header(jose.Header): kid = jose.Field('kid', omitempty=True) url = jose.Field('url', omitempty=True) - @nonce.decoder + # Mypy does not understand the josepy magic happening here, and falsely claims + # that nonce is redefined. Let's ignore the type check here. + @nonce.decoder # type: ignore def nonce(value): # pylint: disable=no-self-argument,missing-function-docstring try: return jose.decode_b64jose(value) @@ -48,7 +50,7 @@ class JWS(jose.JWS): # Per ACME spec, jwk and kid are mutually exclusive, so only include a # jwk field if kid is not provided. include_jwk = kid is None - return super(JWS, cls).sign(payload, key=key, alg=alg, + return super().sign(payload, key=key, alg=alg, protect=frozenset(['nonce', 'url', 'kid', 'jwk', 'alg']), nonce=nonce, url=url, kid=kid, include_jwk=include_jwk) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 388fc4a58..8190fa552 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -1,16 +1,17 @@ -"""Shim class to not have to depend on typing module in prod.""" -import sys +"""Simple shim around the typing module. +This was useful when this code supported Python 2 and typing wasn't always +available. This code is being kept for now for backwards compatibility. -class TypingClass(object): +""" +import warnings +from typing import * # pylint: disable=wildcard-import, unused-wildcard-import +from typing import Collection, IO # type: ignore + +warnings.warn("acme.magic_typing is deprecated and will be removed in a future release.", + DeprecationWarning) + +class TypingClass: """Ignore import errors by getting anything""" def __getattr__(self, name): - return None - -try: - # mypy doesn't respect modifying sys.modules - from typing import * # pylint: disable=wildcard-import, unused-wildcard-import - from typing import Collection, IO # type: ignore -except ImportError: - # mypy complains because TypingClass is not a module - sys.modules[__name__] = TypingClass() # type: ignore + return None # pragma: no cover diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 038cda04b..36207dba0 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,5 +1,9 @@ """ACME protocol messages.""" +from collections.abc import Hashable import json +from typing import Any +from typing import Dict +from typing import Type import josepy as jose @@ -10,13 +14,6 @@ from acme import jws from acme import util from acme.mixins import ResourceMixin -try: - from collections.abc import Hashable -except ImportError: # pragma: no cover - from collections import Hashable - - - OLD_ERROR_PREFIX = "urn:acme:error:" ERROR_PREFIX = "urn:ietf:params:acme:error:" @@ -93,7 +90,9 @@ class Error(jose.JSONObjectWithFields, errors.Error): raise ValueError("The supplied code: %s is not a known ACME error" " code" % code) typ = ERROR_PREFIX + code - return cls(typ=typ, **kwargs) + # Mypy will not understand that the Error constructor accepts a named argument + # "typ" because of josepy magic. Let's ignore the type check here. + return cls(typ=typ, **kwargs) # type: ignore @property def description(self): @@ -130,10 +129,10 @@ class Error(jose.JSONObjectWithFields, errors.Error): class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore """ACME constant.""" __slots__ = ('name',) - POSSIBLE_NAMES = NotImplemented + POSSIBLE_NAMES: Dict[str, '_Constant'] = NotImplemented def __init__(self, name): - super(_Constant, self).__init__() + super().__init__() self.POSSIBLE_NAMES[name] = self # pylint: disable=unsupported-assignment-operation self.name = name @@ -156,13 +155,10 @@ class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore def __hash__(self): return hash((self.__class__, self.name)) - def __ne__(self, other): - return not self == other - class Status(_Constant): """ACME "status" field.""" - POSSIBLE_NAMES = {} # type: dict + POSSIBLE_NAMES: dict = {} STATUS_UNKNOWN = Status('unknown') STATUS_PENDING = Status('pending') STATUS_PROCESSING = Status('processing') @@ -175,7 +171,7 @@ STATUS_DEACTIVATED = Status('deactivated') class IdentifierType(_Constant): """ACME identifier type.""" - POSSIBLE_NAMES = {} # type: dict + POSSIBLE_NAMES: Dict[str, 'IdentifierType'] = {} IDENTIFIER_FQDN = IdentifierType('dns') # IdentifierDNS in Boulder @@ -193,7 +189,7 @@ class Identifier(jose.JSONObjectWithFields): class Directory(jose.JSONDeSerializable): """Directory.""" - _REGISTERED_TYPES = {} # type: dict + _REGISTERED_TYPES: Dict[str, Type[Any]] = {} class Meta(jose.JSONObjectWithFields): """Directory Meta.""" @@ -205,7 +201,7 @@ class Directory(jose.JSONDeSerializable): def __init__(self, **kwargs): kwargs = {self._internal_name(k): v for k, v in kwargs.items()} - super(Directory.Meta, self).__init__(**kwargs) + super().__init__(**kwargs) @property def terms_of_service(self): @@ -215,7 +211,7 @@ class Directory(jose.JSONDeSerializable): def __iter__(self): # When iterating over fields, use the external name 'terms_of_service' instead of # the internal '_terms_of_service'. - for name in super(Directory.Meta, self).__iter__(): + for name in super().__iter__(): yield name[1:] if name == '_terms_of_service' else name def _internal_name(self, name): @@ -227,7 +223,7 @@ class Directory(jose.JSONDeSerializable): return getattr(key, 'resource_type', key) @classmethod - def register(cls, resource_body_cls): + def register(cls, resource_body_cls: Type[Any]) -> Type[Any]: """Register resource.""" resource_type = resource_body_cls.resource_type assert resource_type not in cls._REGISTERED_TYPES @@ -283,7 +279,7 @@ class ResourceBody(jose.JSONObjectWithFields): """ACME Resource Body.""" -class ExternalAccountBinding(object): +class ExternalAccountBinding: """ACME External Account Binding""" @classmethod @@ -361,7 +357,7 @@ class Registration(ResourceBody): if 'contact' in kwargs: # Avoid the __setattr__ used by jose.TypedJSONObjectWithFields object.__setattr__(self, '_add_contact', True) - super(Registration, self).__init__(**kwargs) + super().__init__(**kwargs) def _filter_contact(self, prefix): return tuple( @@ -387,12 +383,12 @@ class Registration(ResourceBody): def to_partial_json(self): """Modify josepy.JSONDeserializable.to_partial_json()""" - jobj = super(Registration, self).to_partial_json() + jobj = super().to_partial_json() return self._add_contact_if_appropriate(jobj) def fields_to_partial_json(self): """Modify josepy.JSONObjectWithFields.fields_to_partial_json()""" - jobj = super(Registration, self).fields_to_partial_json() + jobj = super().fields_to_partial_json() return self._add_contact_if_appropriate(jobj) @property @@ -464,19 +460,19 @@ class ChallengeBody(ResourceBody): def __init__(self, **kwargs): kwargs = {self._internal_name(k): v for k, v in kwargs.items()} - super(ChallengeBody, self).__init__(**kwargs) + super().__init__(**kwargs) def encode(self, name): - return super(ChallengeBody, self).encode(self._internal_name(name)) + return super().encode(self._internal_name(name)) def to_partial_json(self): - jobj = super(ChallengeBody, self).to_partial_json() + jobj = super().to_partial_json() jobj.update(self.chall.to_partial_json()) return jobj @classmethod def fields_from_json(cls, jobj): - jobj_fields = super(ChallengeBody, cls).fields_from_json(jobj) + jobj_fields = super().fields_from_json(jobj) jobj_fields['chall'] = challenges.Challenge.from_json(jobj) return jobj_fields @@ -491,7 +487,7 @@ class ChallengeBody(ResourceBody): def __iter__(self): # When iterating over fields, use the external name 'uri' instead of # the internal '_uri'. - for name in super(ChallengeBody, self).__iter__(): + for name in super().__iter__(): yield name[1:] if name == '_uri' else name def _internal_name(self, name): @@ -537,7 +533,9 @@ class Authorization(ResourceBody): expires = fields.RFC3339Field('expires', omitempty=True) wildcard = jose.Field('wildcard', omitempty=True) - @challenges.decoder + # Mypy does not understand the josepy magic happening here, and falsely claims + # that challenge is redefined. Let's ignore the type check here. + @challenges.decoder # type: ignore def challenges(value): # pylint: disable=no-self-argument,missing-function-docstring return tuple(ChallengeBody.from_json(chall) for chall in value) @@ -636,7 +634,9 @@ class Order(ResourceBody): expires = fields.RFC3339Field('expires', omitempty=True) error = jose.Field('error', omitempty=True, decoder=Error.from_json) - @identifiers.decoder + # Mypy does not understand the josepy magic happening here, and falsely claims + # that identifiers is redefined. Let's ignore the type check here. + @identifiers.decoder # type: ignore def identifiers(value): # pylint: disable=no-self-argument,missing-function-docstring return tuple(Identifier.from_json(identifier) for identifier in value) diff --git a/acme/acme/mixins.py b/acme/acme/mixins.py index 1cd050ccc..6d58e0889 100644 --- a/acme/acme/mixins.py +++ b/acme/acme/mixins.py @@ -1,7 +1,7 @@ """Useful mixins for Challenge and Resource objects""" -class VersionedLEACMEMixin(object): +class VersionedLEACMEMixin: """This mixin stores the version of Let's Encrypt's endpoint being used.""" @property def le_acme_version(self): @@ -20,7 +20,7 @@ class VersionedLEACMEMixin(object): # Required for @property to operate properly. See comment above. object.__setattr__(self, key, value) else: - super(VersionedLEACMEMixin, self).__setattr__(key, value) # pragma: no cover + super().__setattr__(key, value) # pragma: no cover class ResourceMixin(VersionedLEACMEMixin): @@ -30,12 +30,12 @@ class ResourceMixin(VersionedLEACMEMixin): """ def to_partial_json(self): """See josepy.JSONDeserializable.to_partial_json()""" - return _safe_jobj_compliance(super(ResourceMixin, self), + return _safe_jobj_compliance(super(), 'to_partial_json', 'resource') def fields_to_partial_json(self): """See josepy.JSONObjectWithFields.fields_to_partial_json()""" - return _safe_jobj_compliance(super(ResourceMixin, self), + return _safe_jobj_compliance(super(), 'fields_to_partial_json', 'resource') @@ -46,12 +46,12 @@ class TypeMixin(VersionedLEACMEMixin): """ def to_partial_json(self): """See josepy.JSONDeserializable.to_partial_json()""" - return _safe_jobj_compliance(super(TypeMixin, self), + return _safe_jobj_compliance(super(), 'to_partial_json', 'type') def fields_to_partial_json(self): """See josepy.JSONObjectWithFields.fields_to_partial_json()""" - return _safe_jobj_compliance(super(TypeMixin, self), + return _safe_jobj_compliance(super(), 'fields_to_partial_json', 'type') diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 94397f0de..eda45304c 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -7,10 +7,10 @@ import logging import socket import socketserver import threading +from typing import List from acme import challenges from acme import crypto_util -from acme.magic_typing import List logger = logging.getLogger(__name__) @@ -53,7 +53,7 @@ class ACMEServerMixin: allow_reuse_address = True -class BaseDualNetworkedServers(object): +class BaseDualNetworkedServers: """Base class for a pair of IPv6 and IPv4 servers that tries to do everything it's asked for both servers, but where failures in one server don't affect the other. @@ -63,8 +63,8 @@ class BaseDualNetworkedServers(object): def __init__(self, ServerClass, server_address, *remaining_args, **kwargs): port = server_address[1] - self.threads = [] # type: List[threading.Thread] - self.servers = [] # type: List[ACMEServerMixin] + self.threads: List[threading.Thread] = [] + self.servers: List[socketserver.BaseServer] = [] # Must try True first. # Ubuntu, for example, will fail to bind to IPv4 if we've already bound @@ -203,8 +203,24 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.simple_http_resources = kwargs.pop("simple_http_resources", set()) - self.timeout = kwargs.pop('timeout', 30) + self._timeout = kwargs.pop('timeout', 30) BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + self.server: HTTP01Server + + # In parent class BaseHTTPRequestHandler, 'timeout' is a class-level property but we + # need to define its value during the initialization phase in HTTP01RequestHandler. + # However MyPy does not appreciate that we dynamically shadow a class-level property + # with an instance-level property (eg. self.timeout = ... in __init__()). So to make + # everyone happy, we statically redefine 'timeout' as a method property, and set the + # timeout value in a new internal instance-level property _timeout. + @property + def timeout(self): + """ + The default timeout this server should apply to requests. + :return: timeout to apply + :rtype: int + """ + return self._timeout def log_message(self, format, *args): # pylint: disable=redefined-builtin """Log arbitrary message.""" diff --git a/acme/setup.py b/acme/setup.py index 847ca9299..6fa49dafc 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -37,7 +37,7 @@ setup( description='ACME protocol implementation in Python', url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index cc604b0de..71bf61976 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -292,7 +292,7 @@ class TLSALPN01ResponseTest(unittest.TestCase): def test_gen_verify_cert_gen_key(self): cert, key = self.response.gen_cert(self.domain) - self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) + self.assertIsInstance(key, OpenSSL.crypto.PKey) self.assertTrue(self.response.verify_cert(self.domain, cert)) def test_verify_bad_cert(self): @@ -431,7 +431,7 @@ class DNSTest(unittest.TestCase): mock_gen.return_value = mock.sentinel.validation response = self.msg.gen_response(KEY) from acme.challenges import DNSResponse - self.assertTrue(isinstance(response, DNSResponse)) + self.assertIsInstance(response, DNSResponse) self.assertEqual(response.validation, mock.sentinel.validation) def test_validation_domain_name(self): diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 6f9aecda2..35cc0ba25 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -5,6 +5,7 @@ import datetime import http.client as http_client import json import unittest +from typing import Dict from unittest import mock import josepy as jose @@ -61,7 +62,7 @@ class ClientTestBase(unittest.TestCase): self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212') reg = messages.Registration( contact=self.contact, key=KEY.public_key()) - the_arg = dict(reg) # type: Dict + the_arg: Dict = dict(reg) self.new_reg = messages.NewRegistration(**the_arg) self.regr = messages.RegistrationResource( body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') @@ -89,7 +90,7 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" def setUp(self): - super(BackwardsCompatibleClientV2Test, self).setUp() + super().setUp() # contains a loaded cert self.certr = messages.CertificateResource( body=messages_test.CERT) @@ -318,7 +319,7 @@ class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" def setUp(self): - super(ClientTest, self).setUp() + super().setUp() self.directory = DIRECTORY_V1 @@ -603,8 +604,8 @@ class ClientTest(ClientTestBase): # make sure that max_attempts is per-authorization, rather # than global max_attempts=max(len(authzrs[0].retries), len(authzrs[1].retries))) - self.assertTrue(cert[0] is csr) - self.assertTrue(cert[1] is updated_authzrs) + self.assertIs(cert[0], csr) + self.assertIs(cert[1], updated_authzrs) self.assertEqual(updated_authzrs[0].uri, 'a...') self.assertEqual(updated_authzrs[1].uri, 'b.') self.assertEqual(updated_authzrs[0].times, [ @@ -640,7 +641,7 @@ class ClientTest(ClientTestBase): authzr = self.client.deactivate_authorization(self.authzr) self.assertEqual(authzb, authzr.body) self.assertEqual(self.client.net.post.call_count, 1) - self.assertTrue(self.authzr.uri in self.net.post.call_args_list[0][0]) + self.assertIn(self.authzr.uri, self.net.post.call_args_list[0][0]) def test_check_cert(self): self.response.headers['Location'] = self.certr.uri @@ -699,7 +700,7 @@ class ClientTest(ClientTestBase): def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) - self.assertTrue('reason' in obj.to_partial_json().keys()) + self.assertIn('reason', obj.to_partial_json().keys()) self.assertEqual(self.rsn, obj.to_partial_json()['reason']) def test_revoke_bad_status_raises_error(self): @@ -715,7 +716,7 @@ class ClientV2Test(ClientTestBase): """Tests for acme.client.ClientV2.""" def setUp(self): - super(ClientV2Test, self).setUp() + super().setUp() self.directory = DIRECTORY_V2 @@ -876,9 +877,9 @@ class ClientV2Test(ClientTestBase): self.response.headers['Location'] = self.regr.uri self.response.json.return_value = self.regr.body.to_json() self.assertEqual(self.regr, self.client.update_registration(self.regr)) - self.assertNotEqual(self.client.net.account, None) + self.assertIsNotNone(self.client.net.account) self.assertEqual(self.client.net.post.call_count, 2) - self.assertTrue(DIRECTORY_V2.newAccount in self.net.post.call_args_list[0][0]) + self.assertIn(DIRECTORY_V2.newAccount, self.net.post.call_args_list[0][0]) self.response.json.return_value = self.regr.body.update( contact=()).to_json() @@ -942,7 +943,7 @@ class ClientNetworkTest(unittest.TestCase): self.response.links = {} def test_init(self): - self.assertTrue(self.net.verify_ssl is self.verify_ssl) + self.assertIs(self.net.verify_ssl, self.verify_ssl) def test_wrap_in_jws(self): # pylint: disable=protected-access @@ -1184,7 +1185,7 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): def send_request(*args, **kwargs): # pylint: disable=unused-argument,missing-docstring - self.assertFalse("new_nonce_url" in kwargs) + self.assertNotIn("new_nonce_url", kwargs) method = args[0] uri = args[1] if method == 'HEAD' and uri != "new_nonce_uri": @@ -1329,7 +1330,7 @@ class ClientNetworkSourceAddressBindingTest(unittest.TestCase): from acme.client import ClientNetwork net = ClientNetwork(key=None, alg=None, source_address=self.source_address) for adapter in net.session.adapters.values(): - self.assertTrue(self.source_address in adapter.source_address) + self.assertIn(self.source_address, adapter.source_address) def test_behavior_assumption(self): """This is a test that guardrails the HTTPAdapter behavior so that if the default for diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index f271ad37d..cc81a2f1f 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -5,6 +5,7 @@ import socketserver import threading import time import unittest +from typing import List import josepy as jose import OpenSSL @@ -180,7 +181,7 @@ class RandomSnTest(unittest.TestCase): def setUp(self): self.cert_count = 5 - self.serial_num = [] # type: List[int] + self.serial_num: List[int] = [] self.key = OpenSSL.crypto.PKey() self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) @@ -190,7 +191,7 @@ class RandomSnTest(unittest.TestCase): for _ in range(self.cert_count): cert = gen_ss_cert(self.key, ['dummy'], force_san=True) self.serial_num.append(cert.get_serial_number()) - self.assertTrue(len(set(self.serial_num)) > 1) + self.assertGreater(len(set(self.serial_num)), 1) class MakeCSRTest(unittest.TestCase): """Test for standalone functions.""" @@ -205,8 +206,8 @@ class MakeCSRTest(unittest.TestCase): def test_make_csr(self): csr_pem = self._call_with_key(["a.example", "b.example"]) - self.assertTrue(b'--BEGIN CERTIFICATE REQUEST--' in csr_pem) - self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) + self.assertIn(b'--BEGIN CERTIFICATE REQUEST--', csr_pem) + self.assertIn(b'--END CERTIFICATE REQUEST--', csr_pem) csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index 11c57059c..f325b284e 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -24,8 +24,8 @@ class MissingNonceTest(unittest.TestCase): self.error = MissingNonce(self.response) def test_str(self): - self.assertTrue("FOO" in str(self.error)) - self.assertTrue("{}" in str(self.error)) + self.assertIn("FOO", str(self.error)) + self.assertIn("{}", str(self.error)) class PollErrorTest(unittest.TestCase): diff --git a/acme/tests/jws_test.py b/acme/tests/jws_test.py index 2e6ad72dd..0787fb340 100644 --- a/acme/tests/jws_test.py +++ b/acme/tests/jws_test.py @@ -48,7 +48,7 @@ class JWSTest(unittest.TestCase): self.assertEqual(jws.signature.combined.nonce, self.nonce) self.assertEqual(jws.signature.combined.url, self.url) self.assertEqual(jws.signature.combined.kid, self.kid) - self.assertEqual(jws.signature.combined.jwk, None) + self.assertIsNone(jws.signature.combined.jwk) # TODO: check that nonce is in protected header self.assertEqual(jws, JWS.from_json(jws.to_json())) @@ -58,7 +58,7 @@ class JWSTest(unittest.TestCase): jws = JWS.sign(payload=b'foo', key=self.privkey, alg=jose.RS256, nonce=self.nonce, url=self.url) - self.assertEqual(jws.signature.combined.kid, None) + self.assertIsNone(jws.signature.combined.kid) self.assertEqual(jws.signature.combined.jwk, self.pubkey) diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 048995916..d470337bd 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -1,6 +1,7 @@ """Tests for acme.magic_typing.""" import sys import unittest +import warnings from unittest import mock @@ -9,32 +10,21 @@ class MagicTypingTest(unittest.TestCase): def test_import_success(self): try: import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover typing_class_mock = mock.MagicMock() text_mock = mock.MagicMock() typing_class_mock.Text = text_mock sys.modules['typing'] = typing_class_mock if 'acme.magic_typing' in sys.modules: - del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text + del sys.modules['acme.magic_typing'] # pragma: no cover + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + from acme.magic_typing import Text self.assertEqual(Text, text_mock) del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing - def test_import_failure(self): - try: - import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover - sys.modules['typing'] = None - if 'acme.magic_typing' in sys.modules: - del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text - self.assertTrue(Text is None) - del sys.modules['acme.magic_typing'] - sys.modules['typing'] = temp_typing - if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index 74d1737ec..3f0f29215 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -1,4 +1,5 @@ """Tests for acme.messages.""" +from typing import Dict import unittest from unittest import mock @@ -40,13 +41,13 @@ class ErrorTest(unittest.TestCase): def test_description(self): self.assertEqual('The request message was malformed', self.error.description) - self.assertTrue(self.error_custom.description is None) + self.assertIsNone(self.error_custom.description) def test_code(self): from acme.messages import Error self.assertEqual('malformed', self.error.code) - self.assertEqual(None, self.error_custom.code) - self.assertEqual(None, Error().code) + self.assertIsNone(self.error_custom.code) + self.assertIsNone(Error().code) def test_is_acme_error(self): from acme.messages import is_acme_error, Error @@ -81,7 +82,7 @@ class ConstantTest(unittest.TestCase): from acme.messages import _Constant class MockConstant(_Constant): # pylint: disable=missing-docstring - POSSIBLE_NAMES = {} # type: Dict + POSSIBLE_NAMES: Dict = {} self.MockConstant = MockConstant # pylint: disable=invalid-name self.const_a = MockConstant('a') @@ -259,10 +260,10 @@ class RegistrationTest(unittest.TestCase): self.assertEqual(empty_new_reg.contact, ()) self.assertEqual(new_reg_with_contact.contact, ()) - self.assertTrue('contact' not in empty_new_reg.to_partial_json()) - self.assertTrue('contact' not in empty_new_reg.fields_to_partial_json()) - self.assertTrue('contact' in new_reg_with_contact.to_partial_json()) - self.assertTrue('contact' in new_reg_with_contact.fields_to_partial_json()) + self.assertNotIn('contact', empty_new_reg.to_partial_json()) + self.assertNotIn('contact', empty_new_reg.fields_to_partial_json()) + self.assertIn('contact', new_reg_with_contact.to_partial_json()) + self.assertIn('contact', new_reg_with_contact.fields_to_partial_json()) class UpdateRegistrationTest(unittest.TestCase): @@ -405,7 +406,7 @@ class AuthorizationResourceTest(unittest.TestCase): authzr = AuthorizationResource( uri=mock.sentinel.uri, body=mock.sentinel.body) - self.assertTrue(isinstance(authzr, jose.JSONDeSerializable)) + self.assertIsInstance(authzr, jose.JSONDeSerializable) class CertificateRequestTest(unittest.TestCase): @@ -416,7 +417,7 @@ class CertificateRequestTest(unittest.TestCase): self.req = CertificateRequest(csr=CSR) def test_json_de_serializable(self): - self.assertTrue(isinstance(self.req, jose.JSONDeSerializable)) + self.assertIsInstance(self.req, jose.JSONDeSerializable) from acme.messages import CertificateRequest self.assertEqual( self.req, CertificateRequest.from_json(self.req.to_json())) @@ -432,7 +433,7 @@ class CertificateResourceTest(unittest.TestCase): cert_chain_uri=mock.sentinel.cert_chain_uri) def test_json_de_serializable(self): - self.assertTrue(isinstance(self.certr, jose.JSONDeSerializable)) + self.assertIsInstance(self.certr, jose.JSONDeSerializable) from acme.messages import CertificateResource self.assertEqual( self.certr, CertificateResource.from_json(self.certr.to_json())) diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index e6aa8f2d6..17d73dba8 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -4,6 +4,7 @@ import socket import socketserver import threading import unittest +from typing import Set from unittest import mock import josepy as jose @@ -41,7 +42,7 @@ class HTTP01ServerTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() # type: Set + self.resources: Set = set() from acme.standalone import HTTP01Server self.server = HTTP01Server(('', 0), resources=self.resources) @@ -218,7 +219,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() # type: Set + self.resources: Set = set() from acme.standalone import HTTP01DualNetworkedServers self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources) diff --git a/certbot-apache/certbot_apache/_internal/apache_util.py b/certbot-apache/certbot_apache/_internal/apache_util.py index 93612f424..9b9855a45 100644 --- a/certbot-apache/certbot_apache/_internal/apache_util.py +++ b/certbot-apache/certbot_apache/_internal/apache_util.py @@ -9,7 +9,6 @@ import pkg_resources from certbot import errors from certbot import util - from certbot.compat import os logger = logging.getLogger(__name__) diff --git a/certbot-apache/certbot_apache/_internal/apacheparser.py b/certbot-apache/certbot_apache/_internal/apacheparser.py index c7b723ae6..d3bd1a4bf 100644 --- a/certbot-apache/certbot_apache/_internal/apacheparser.py +++ b/certbot-apache/certbot_apache/_internal/apacheparser.py @@ -1,4 +1,5 @@ """ apacheconfig implementation of the ParserNode interfaces """ +from typing import Tuple from certbot_apache._internal import assertions from certbot_apache._internal import interfaces @@ -14,14 +15,14 @@ class ApacheParserNode(interfaces.ParserNode): def __init__(self, **kwargs): ancestor, dirty, filepath, metadata = util.parsernode_kwargs(kwargs) # pylint: disable=unused-variable - super(ApacheParserNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.ancestor = ancestor self.filepath = filepath self.dirty = dirty self.metadata = metadata self._raw = self.metadata["ac_ast"] - def save(self, msg): # pragma: no cover + def save(self, msg): # pragma: no cover pass def find_ancestors(self, name): # pylint: disable=unused-variable @@ -38,7 +39,7 @@ class ApacheCommentNode(ApacheParserNode): def __init__(self, **kwargs): comment, kwargs = util.commentnode_kwargs(kwargs) # pylint: disable=unused-variable - super(ApacheCommentNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.comment = comment def __eq__(self, other): # pragma: no cover @@ -56,7 +57,7 @@ class ApacheDirectiveNode(ApacheParserNode): def __init__(self, **kwargs): name, parameters, enabled, kwargs = util.directivenode_kwargs(kwargs) - super(ApacheDirectiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.name = name self.parameters = parameters self.enabled = enabled @@ -82,8 +83,8 @@ class ApacheBlockNode(ApacheDirectiveNode): """ apacheconfig implementation of BlockNode interface """ def __init__(self, **kwargs): - super(ApacheBlockNode, self).__init__(**kwargs) - self.children = () + super().__init__(**kwargs) + self.children: Tuple[ApacheParserNode, ...] = () def __eq__(self, other): # pragma: no cover if isinstance(other, self.__class__): diff --git a/certbot-apache/certbot_apache/_internal/assertions.py b/certbot-apache/certbot_apache/_internal/assertions.py index 2b2ce4f50..53603c526 100644 --- a/certbot-apache/certbot_apache/_internal/assertions.py +++ b/certbot-apache/certbot_apache/_internal/assertions.py @@ -3,7 +3,6 @@ import fnmatch from certbot_apache._internal import interfaces - PASS = "CERTBOT_PASS_ASSERT" diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index 3b2ce40d8..896e17cf8 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -64,10 +64,10 @@ Translates over to: "/files/etc/apache2/apache2.conf/bLoCk[1]", ] """ -from acme.magic_typing import Set +from typing import Set + from certbot import errors from certbot.compat import os - from certbot_apache._internal import apache_util from certbot_apache._internal import assertions from certbot_apache._internal import interfaces @@ -80,7 +80,7 @@ class AugeasParserNode(interfaces.ParserNode): def __init__(self, **kwargs): ancestor, dirty, filepath, metadata = util.parsernode_kwargs(kwargs) # pylint: disable=unused-variable - super(AugeasParserNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.ancestor = ancestor self.filepath = filepath self.dirty = dirty @@ -169,7 +169,7 @@ class AugeasCommentNode(AugeasParserNode): def __init__(self, **kwargs): comment, kwargs = util.commentnode_kwargs(kwargs) # pylint: disable=unused-variable - super(AugeasCommentNode, self).__init__(**kwargs) + super().__init__(**kwargs) # self.comment = comment self.comment = comment @@ -188,7 +188,7 @@ class AugeasDirectiveNode(AugeasParserNode): def __init__(self, **kwargs): name, parameters, enabled, kwargs = util.directivenode_kwargs(kwargs) - super(AugeasDirectiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.name = name self.enabled = enabled if parameters: @@ -245,7 +245,7 @@ class AugeasBlockNode(AugeasDirectiveNode): """ Augeas implementation of BlockNode interface """ def __init__(self, **kwargs): - super(AugeasBlockNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.children = () def __eq__(self, other): @@ -355,7 +355,7 @@ class AugeasBlockNode(AugeasDirectiveNode): ownpath = self.metadata.get("augeaspath") directives = self.parser.find_dir(name, start=ownpath, exclude=exclude) - already_parsed = set() # type: Set[str] + already_parsed: Set[str] = set() for directive in directives: # Remove the /arg part from the Augeas path directive = directive.partition("/arg")[0] diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 16def1998..abf199793 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -1,28 +1,24 @@ """Apache Configurator.""" # pylint: disable=too-many-lines from collections import defaultdict -from distutils.version import LooseVersion import copy +from distutils.version import LooseVersion import fnmatch import logging import re import socket import time +from typing import DefaultDict +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Union import zope.component import zope.interface -try: - import apacheconfig - HAS_APACHECONFIG = True -except ImportError: # pragma: no cover - HAS_APACHECONFIG = False from acme import challenges -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Union from certbot import errors from certbot import interfaces from certbot import util @@ -40,10 +36,61 @@ from certbot_apache._internal import dualparser from certbot_apache._internal import http_01 from certbot_apache._internal import obj from certbot_apache._internal import parser +from certbot_apache._internal.dualparser import DualBlockNode +from certbot_apache._internal.obj import VirtualHost +from certbot_apache._internal.parser import ApacheParser + +try: + import apacheconfig + HAS_APACHECONFIG = True +except ImportError: # pragma: no cover + HAS_APACHECONFIG = False + logger = logging.getLogger(__name__) +class OsOptions: + """ + Dedicated class to describe the OS specificities (eg. paths, binary names) + that the Apache configurator needs to be aware to operate properly. + """ + def __init__(self, + server_root="/etc/apache2", + vhost_root="/etc/apache2/sites-available", + vhost_files="*", + logs_root="/var/log/apache2", + ctl="apache2ctl", + version_cmd: Optional[List[str]] = None, + restart_cmd: Optional[List[str]] = None, + restart_cmd_alt: Optional[List[str]] = None, + conftest_cmd: Optional[List[str]] = None, + enmod: Optional[str] = None, + dismod: Optional[str] = None, + le_vhost_ext="-le-ssl.conf", + handle_modules=False, + handle_sites=False, + challenge_location="/etc/apache2", + apache_bin: Optional[str] = None, + ): + self.server_root = server_root + self.vhost_root = vhost_root + self.vhost_files = vhost_files + self.logs_root = logs_root + self.ctl = ctl + self.version_cmd = ['apache2ctl', '-v'] if not version_cmd else version_cmd + self.restart_cmd = ['apache2ctl', 'graceful'] if not restart_cmd else restart_cmd + self.restart_cmd_alt = restart_cmd_alt + self.conftest_cmd = ['apache2ctl', 'configtest'] if not conftest_cmd else conftest_cmd + self.enmod = enmod + self.dismod = dismod + self.le_vhost_ext = le_vhost_ext + self.handle_modules = handle_modules + self.handle_sites = handle_sites + self.challenge_location = challenge_location + self.bin = apache_bin + + # TODO: Augeas sections ie. , beginning and closing # tags need to be the same case, otherwise Augeas doesn't recognize them. # This is not able to be completely remedied by regular expressions because @@ -99,27 +146,7 @@ class ApacheConfigurator(common.Installer): " change depending on the operating system Certbot is run on.)" ) - OS_DEFAULTS = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/sites-available", - vhost_files="*", - logs_root="/var/log/apache2", - ctl="apache2ctl", - version_cmd=['apache2ctl', '-v'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, - challenge_location="/etc/apache2", - bin=None - ) - - def option(self, key): - """Get a value from options""" - return self.options.get(key) + OS_DEFAULTS = OsOptions() def pick_apache_config(self, warn_on_no_mod_ssl=True): """ @@ -149,14 +176,14 @@ class ApacheConfigurator(common.Installer): for o in opts: # Config options use dashes instead of underscores if self.conf(o.replace("_", "-")) is not None: - self.options[o] = self.conf(o.replace("_", "-")) + setattr(self.options, o, self.conf(o.replace("_", "-"))) else: - self.options[o] = self.OS_DEFAULTS[o] + setattr(self.options, o, getattr(self.OS_DEFAULTS, o)) # Special cases - self.options["version_cmd"][0] = self.option("ctl") - self.options["restart_cmd"][0] = self.option("ctl") - self.options["conftest_cmd"][0] = self.option("ctl") + self.options.version_cmd[0] = self.options.ctl + self.options.restart_cmd[0] = self.options.ctl + self.options.conftest_cmd[0] = self.options.ctl @classmethod def add_parser_arguments(cls, add): @@ -171,30 +198,30 @@ class ApacheConfigurator(common.Installer): else: # cls.OS_DEFAULTS can be distribution specific, see override classes DEFAULTS = cls.OS_DEFAULTS - add("enmod", default=DEFAULTS["enmod"], + add("enmod", default=DEFAULTS.enmod, help="Path to the Apache 'a2enmod' binary") - add("dismod", default=DEFAULTS["dismod"], + add("dismod", default=DEFAULTS.dismod, help="Path to the Apache 'a2dismod' binary") - add("le-vhost-ext", default=DEFAULTS["le_vhost_ext"], + add("le-vhost-ext", default=DEFAULTS.le_vhost_ext, help="SSL vhost configuration extension") - add("server-root", default=DEFAULTS["server_root"], + add("server-root", default=DEFAULTS.server_root, help="Apache server root directory") add("vhost-root", default=None, help="Apache server VirtualHost configuration root") - add("logs-root", default=DEFAULTS["logs_root"], + add("logs-root", default=DEFAULTS.logs_root, help="Apache server logs directory") add("challenge-location", - default=DEFAULTS["challenge_location"], + default=DEFAULTS.challenge_location, help="Directory path for challenge configuration") - add("handle-modules", default=DEFAULTS["handle_modules"], + add("handle-modules", default=DEFAULTS.handle_modules, help="Let installer handle enabling required modules for you " + "(Only Ubuntu/Debian currently)") - add("handle-sites", default=DEFAULTS["handle_sites"], + add("handle-sites", default=DEFAULTS.handle_sites, help="Let installer handle enabling sites for you " + "(Only Ubuntu/Debian currently)") - add("ctl", default=DEFAULTS["ctl"], + add("ctl", default=DEFAULTS.ctl, help="Full path to Apache control script") - add("bin", default=DEFAULTS["bin"], + add("bin", default=DEFAULTS.bin, help="Full path to apache2/httpd binary") def __init__(self, *args, **kwargs): @@ -207,33 +234,33 @@ class ApacheConfigurator(common.Installer): version = kwargs.pop("version", None) use_parsernode = kwargs.pop("use_parsernode", False) openssl_version = kwargs.pop("openssl_version", None) - super(ApacheConfigurator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) # Add name_server association dict - self.assoc = {} # type: Dict[str, obj.VirtualHost] + self.assoc: Dict[str, obj.VirtualHost] = {} # Outstanding challenges - self._chall_out = set() # type: Set[KeyAuthorizationAnnotatedChallenge] + self._chall_out: Set[KeyAuthorizationAnnotatedChallenge] = set() # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {} # Maps enhancements to vhosts we've enabled the enhancement for - self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] + self._enhanced_vhosts: DefaultDict[str, Set[obj.VirtualHost]] = defaultdict(set) # Temporary state for AutoHSTS enhancement - self._autohsts = {} # type: Dict[str, Dict[str, Union[int, float]]] + self._autohsts: Dict[str, Dict[str, Union[int, float]]] = {} # Reverter save notes self.save_notes = "" # Should we use ParserNode implementation instead of the old behavior self.USE_PARSERNODE = use_parsernode # Saves the list of file paths that were parsed initially, and # not added to parser tree by self.conf("vhost-root") for example. - self.parsed_paths = [] # type: List[str] + self.parsed_paths: List[str] = [] # These will be set in the prepare function self._prepared = False - self.parser = None - self.parser_root = None + self.parser: ApacheParser + self.parser_root: Optional[DualBlockNode] = None self.version = version self._openssl_version = openssl_version - self.vhosts = None + self.vhosts: List[VirtualHost] self.options = copy.deepcopy(self.OS_DEFAULTS) self._enhance_func = {"redirect": self._enable_redirect, "ensure-http-header": self._set_http_header, @@ -283,8 +310,8 @@ class ApacheConfigurator(common.Installer): ssl_module_location = self.parser.standard_path_from_server_root(ssl_module_location) else: # Possibility B: ssl_module is statically linked into Apache - if self.option("bin"): - ssl_module_location = self.option("bin") + if self.options.bin: + ssl_module_location = self.options.bin else: logger.warning("ssl_module is statically linked but --apache-bin is " "missing; not disabling session tickets.") @@ -314,7 +341,7 @@ class ApacheConfigurator(common.Installer): self._prepare_options() # Verify Apache is installed - self._verify_exe_availability(self.option("ctl")) + self._verify_exe_availability(self.options.ctl) # Make sure configuration is valid self.config_test() @@ -342,8 +369,9 @@ class ApacheConfigurator(common.Installer): "augeaspath": self.parser.get_root_augpath(), "ac_ast": None} if self.USE_PARSERNODE: - self.parser_root = self.get_parsernode_root(pn_meta) - self.parsed_paths = self.parser_root.parsed_paths() + parser_root = self.get_parsernode_root(pn_meta) + self.parser_root = parser_root + self.parsed_paths = parser_root.parsed_paths() # Check for errors in parsing files with Augeas self.parser.check_parsing_errors("httpd.aug") @@ -353,20 +381,20 @@ class ApacheConfigurator(common.Installer): # We may try to enable mod_ssl later. If so, we shouldn't warn if we can't find it now. # This is currently only true for debian/ubuntu. - warn_on_no_mod_ssl = not self.option("handle_modules") + warn_on_no_mod_ssl = not self.options.handle_modules self.install_ssl_options_conf(self.mod_ssl_conf, self.updated_mod_ssl_conf_digest, warn_on_no_mod_ssl) # Prevent two Apache plugins from modifying a config at once try: - util.lock_dir_until_exit(self.option("server_root")) + util.lock_dir_until_exit(self.options.server_root) except (OSError, errors.LockError): logger.debug("Encountered error:", exc_info=True) raise errors.PluginError( "Unable to create a lock file in {0}. Are you running" " Certbot with sufficient privileges to modify your" - " Apache configuration?".format(self.option("server_root"))) + " Apache configuration?".format(self.options.server_root)) self._prepared = True def save(self, title=None, temporary=False): @@ -402,10 +430,10 @@ class ApacheConfigurator(common.Installer): :raises .errors.PluginError: If unable to recover the configuration """ - super(ApacheConfigurator, self).recovery_routine() + super().recovery_routine() # Reload configuration after these changes take effect if needed # ie. ApacheParser has been initialized. - if self.parser: + if hasattr(self, "parser"): # TODO: wrap into non-implementation specific parser interface self.parser.aug.load() @@ -427,7 +455,7 @@ class ApacheConfigurator(common.Installer): the function is unable to correctly revert the configuration """ - super(ApacheConfigurator, self).rollback_checkpoints(rollback) + super().rollback_checkpoints(rollback) self.parser.aug.load() def _verify_exe_availability(self, exe): @@ -441,7 +469,7 @@ class ApacheConfigurator(common.Installer): """Initializes the ApacheParser""" # If user provided vhost_root value in command line, use it return parser.ApacheParser( - self.option("server_root"), self.conf("vhost-root"), + self.options.server_root, self.conf("vhost-root"), self.version, configurator=self) def get_parsernode_root(self, metadata): @@ -449,9 +477,9 @@ class ApacheConfigurator(common.Installer): if HAS_APACHECONFIG: apache_vars = {} - apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) - apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) - apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) + apache_vars["defines"] = apache_util.parse_defines(self.options.ctl) + apache_vars["includes"] = apache_util.parse_includes(self.options.ctl) + apache_vars["modules"] = apache_util.parse_modules(self.options.ctl) metadata["apache_vars"] = apache_vars with open(self.parser.loc["root"]) as f: @@ -832,7 +860,7 @@ class ApacheConfigurator(common.Installer): :rtype: set """ - all_names = set() # type: Set[str] + all_names: Set[str] = set() vhost_macro = [] @@ -996,8 +1024,8 @@ class ApacheConfigurator(common.Installer): """ # Search base config, and all included paths for VirtualHosts - file_paths = {} # type: Dict[str, str] - internal_paths = defaultdict(set) # type: DefaultDict[str, Set[str]] + file_paths: Dict[str, str] = {} + internal_paths: DefaultDict[str, Set[str]] = defaultdict(set) vhs = [] # Make a list of parser paths because the parser_paths # dictionary may be modified during the loop. @@ -1048,6 +1076,9 @@ class ApacheConfigurator(common.Installer): :rtype: list """ + if not self.parser_root: + raise errors.Error("This ApacheConfigurator instance is not" # pragma: no cover + " configured to use a node parser.") vhs = [] vhosts = self.parser_root.find_blocks("VirtualHost", exclude=False) for vhblock in vhosts: @@ -1300,7 +1331,7 @@ class ApacheConfigurator(common.Installer): :param boolean temp: If the change is temporary """ - if self.option("handle_modules"): + if self.options.handle_modules: if self.version >= (2, 4) and ("socache_shmcb_module" not in self.parser.modules): self.enable_mod("socache_shmcb", temp=temp) @@ -1320,7 +1351,7 @@ class ApacheConfigurator(common.Installer): Duplicates vhost and adds default ssl options New vhost will reside as (nonssl_vhost.path) + - ``self.option("le_vhost_ext")`` + ``self.options.le_vhost_ext`` .. note:: This function saves the configuration @@ -1419,15 +1450,15 @@ class ApacheConfigurator(common.Installer): """ if self.conf("vhost-root") and os.path.exists(self.conf("vhost-root")): - fp = os.path.join(filesystem.realpath(self.option("vhost_root")), + fp = os.path.join(filesystem.realpath(self.options.vhost_root), os.path.basename(non_ssl_vh_fp)) else: # Use non-ssl filepath fp = filesystem.realpath(non_ssl_vh_fp) if fp.endswith(".conf"): - return fp[:-(len(".conf"))] + self.option("le_vhost_ext") - return fp + self.option("le_vhost_ext") + return fp[:-(len(".conf"))] + self.options.le_vhost_ext + return fp + self.options.le_vhost_ext def _sift_rewrite_rule(self, line): """Decides whether a line should be copied to a SSL vhost. @@ -2156,7 +2187,7 @@ class ApacheConfigurator(common.Installer): # There can be other RewriteRule directive lines in vhost config. # rewrite_args_dict keys are directive ids and the corresponding value # for each is a list of arguments to that directive. - rewrite_args_dict = defaultdict(list) # type: DefaultDict[str, List[str]] + rewrite_args_dict: DefaultDict[str, List[str]] = defaultdict(list) pat = r'(.*directive\[\d+\]).*' for match in rewrite_path: m = re.match(pat, match) @@ -2250,7 +2281,7 @@ class ApacheConfigurator(common.Installer): if ssl_vhost.aliases: serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases) - rewrite_rule_args = [] # type: List[str] + rewrite_rule_args: List[str] = [] if self.get_version() >= (2, 3, 9): rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END else: @@ -2271,7 +2302,7 @@ class ApacheConfigurator(common.Installer): addr in self._get_proposed_addrs(ssl_vhost)), servername, serveralias, " ".join(rewrite_rule_args), - self.option("logs_root"))) + self.options.logs_root)) def _write_out_redirect(self, ssl_vhost, text): # This is the default name @@ -2283,7 +2314,7 @@ class ApacheConfigurator(common.Installer): if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name - redirect_filepath = os.path.join(self.option("vhost_root"), + redirect_filepath = os.path.join(self.options.vhost_root, redirect_filename) # Register the new file that will be created @@ -2403,19 +2434,18 @@ class ApacheConfigurator(common.Installer): """ try: - util.run_script(self.option("restart_cmd")) + util.run_script(self.options.restart_cmd) except errors.SubprocessError as err: logger.info("Unable to restart apache using %s", - self.option("restart_cmd")) - alt_restart = self.option("restart_cmd_alt") + self.options.restart_cmd) + alt_restart = self.options.restart_cmd_alt if alt_restart: logger.debug("Trying alternative restart command: %s", alt_restart) # There is an alternative restart command available # This usually is "restart" verb while original is "graceful" try: - util.run_script(self.option( - "restart_cmd_alt")) + util.run_script(self.options.restart_cmd_alt) return except errors.SubprocessError as secerr: error = str(secerr) @@ -2430,7 +2460,7 @@ class ApacheConfigurator(common.Installer): """ try: - util.run_script(self.option("conftest_cmd")) + util.run_script(self.options.conftest_cmd) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -2446,11 +2476,11 @@ class ApacheConfigurator(common.Installer): """ try: - stdout, _ = util.run_script(self.option("version_cmd")) + stdout, _ = util.run_script(self.options.version_cmd) except errors.SubprocessError: raise errors.PluginError( "Unable to run %s -v" % - self.option("version_cmd")) + self.options.version_cmd) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) matches = regex.findall(stdout) diff --git a/certbot-apache/certbot_apache/_internal/dualparser.py b/certbot-apache/certbot_apache/_internal/dualparser.py index eef8f2a0e..c89ff95be 100644 --- a/certbot-apache/certbot_apache/_internal/dualparser.py +++ b/certbot-apache/certbot_apache/_internal/dualparser.py @@ -1,10 +1,10 @@ """ Dual ParserNode implementation """ +from certbot_apache._internal import apacheparser from certbot_apache._internal import assertions from certbot_apache._internal import augeasparser -from certbot_apache._internal import apacheparser -class DualNodeBase(object): +class DualNodeBase: """ Dual parser interface for in development testing. This is used as the base class for dual parser interface classes. This class handles runtime attribute value assertions.""" diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 5ef44fa2e..83a1a8e08 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -1,9 +1,9 @@ """A class that performs HTTP-01 challenges for Apache""" -import logging import errno +import logging +from typing import List +from typing import Set -from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot.compat import filesystem from certbot.compat import os @@ -47,7 +47,7 @@ class ApacheHttp01(common.ChallengePerformer): """ def __init__(self, *args, **kwargs): - super(ApacheHttp01, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.challenge_conf_pre = os.path.join( self.configurator.conf("challenge-location"), "le_http_01_challenge_pre.conf") @@ -57,7 +57,7 @@ class ApacheHttp01(common.ChallengePerformer): self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") - self.moded_vhosts = set() # type: Set[VirtualHost] + self.moded_vhosts: Set[VirtualHost] = set() def perform(self): """Perform all HTTP-01 challenges.""" @@ -93,7 +93,7 @@ class ApacheHttp01(common.ChallengePerformer): self.configurator.enable_mod(mod, temp=True) def _mod_config(self): - selected_vhosts = [] # type: List[VirtualHost] + selected_vhosts: List[VirtualHost] = [] http_port = str(self.configurator.config.http01_port) for chall in self.achalls: # Search for matching VirtualHosts diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 77daa5b56..106e2778a 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -102,7 +102,6 @@ For this reason the internal representation of data should not ignore the case. import abc - class ParserNode(object, metaclass=abc.ABCMeta): """ ParserNode is the basic building block of the tree of such nodes, @@ -239,7 +238,7 @@ class CommentNode(ParserNode, metaclass=abc.ABCMeta): created or changed after the last save. Default: False. :type dirty: bool """ - super(CommentNode, self).__init__(ancestor=kwargs['ancestor'], + super().__init__(ancestor=kwargs['ancestor'], dirty=kwargs.get('dirty', False), filepath=kwargs['filepath'], metadata=kwargs.get('metadata', {})) # pragma: no cover @@ -303,7 +302,7 @@ class DirectiveNode(ParserNode, metaclass=abc.ABCMeta): :type enabled: bool """ - super(DirectiveNode, self).__init__(ancestor=kwargs['ancestor'], + super().__init__(ancestor=kwargs['ancestor'], dirty=kwargs.get('dirty', False), filepath=kwargs['filepath'], metadata=kwargs.get('metadata', {})) # pragma: no cover diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 498766744..9001a860d 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -1,7 +1,7 @@ """Module contains classes used by the Apache Configurator.""" import re +from typing import Set -from acme.magic_typing import Set from certbot.plugins import common @@ -20,16 +20,13 @@ class Addr(common.Addr): self.is_wildcard() and other.is_wildcard())) return False - def __ne__(self, other): - return not self.__eq__(other) - def __repr__(self): return "certbot_apache._internal.obj.Addr(" + repr(self.tup) + ")" def __hash__(self): # pylint: disable=useless-super-delegation # Python 3 requires explicit overridden for __hash__ if __eq__ or # __cmp__ is overridden. See https://bugs.python.org/issue2235 - return super(Addr, self).__hash__() + return super().__hash__() def _addr_less_specific(self, addr): """Returns if addr.get_addr() is more specific than self.get_addr().""" @@ -98,7 +95,7 @@ class Addr(common.Addr): return self.get_addr_obj(port) -class VirtualHost(object): +class VirtualHost: """Represents an Apache Virtualhost. :ivar str filep: file path of VH @@ -140,7 +137,7 @@ class VirtualHost(object): def get_names(self): """Return a set of all names.""" - all_names = set() # type: Set[str] + all_names: Set[str] = set() all_names.update(self.aliases) # Strip out any scheme:// and field from servername if self.name is not None: @@ -191,9 +188,6 @@ class VirtualHost(object): return False - def __ne__(self, other): - return not self.__eq__(other) - def __hash__(self): return hash((self.filep, self.path, tuple(self.addrs), tuple(self.get_names()), @@ -251,7 +245,7 @@ class VirtualHost(object): # already_found acts to keep everything very conservative. # Don't allow multiple ip:ports in same set. - already_found = set() # type: Set[str] + already_found: Set[str] = set() for addr in vhost.addrs: for local_addr in self.addrs: diff --git a/certbot-apache/certbot_apache/_internal/override_arch.py b/certbot-apache/certbot_apache/_internal/override_arch.py index 1c3aed1dc..30d161a4e 100644 --- a/certbot-apache/certbot_apache/_internal/override_arch.py +++ b/certbot-apache/certbot_apache/_internal/override_arch.py @@ -3,13 +3,14 @@ import zope.interface from certbot import interfaces from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class ArchConfigurator(configurator.ApacheConfigurator): """Arch Linux specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/httpd", vhost_root="/etc/httpd/conf", vhost_files="*.conf", @@ -18,11 +19,5 @@ class ArchConfigurator(configurator.ApacheConfigurator): version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/httpd/conf", - bin=None, ) diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 9b2ee54c9..c1a69885c 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -1,9 +1,10 @@ """ Distribution specific override class for CentOS family (RHEL, Fedora) """ import logging +from typing import cast +from typing import List import zope.interface -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot import util @@ -11,6 +12,7 @@ from certbot.errors import MisconfigurationError from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser +from certbot_apache._internal.configurator import OsOptions logger = logging.getLogger(__name__) @@ -19,7 +21,7 @@ logger = logging.getLogger(__name__) class CentOSConfigurator(configurator.ApacheConfigurator): """CentOS specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/httpd", vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", @@ -29,13 +31,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): restart_cmd=['apachectl', 'graceful'], restart_cmd_alt=['apachectl', 'restart'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/httpd/conf.d", - bin=None, ) def config_test(self): @@ -50,7 +46,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): fedora = os_info[0].lower() == "fedora" try: - super(CentOSConfigurator, self).config_test() + super().config_test() except errors.MisconfigurationError: if fedora: self._try_restart_fedora() @@ -68,20 +64,22 @@ class CentOSConfigurator(configurator.ApacheConfigurator): raise errors.MisconfigurationError(str(err)) # Finish with actual config check to see if systemctl restart helped - super(CentOSConfigurator, self).config_test() + super().config_test() def _prepare_options(self): """ Override the options dictionary initialization in order to support alternative restart cmd used in CentOS. """ - super(CentOSConfigurator, self)._prepare_options() - self.options["restart_cmd_alt"][0] = self.option("ctl") + super()._prepare_options() + if not self.options.restart_cmd_alt: # pragma: no cover + raise ValueError("OS option restart_cmd_alt must be set for CentOS.") + self.options.restart_cmd_alt[0] = self.options.ctl def get_parser(self): """Initializes the ApacheParser""" return CentOSParser( - self.option("server_root"), self.option("vhost_root"), + self.options.server_root, self.options.vhost_root, self.version, configurator=self) def _deploy_cert(self, *args, **kwargs): # pylint: disable=arguments-differ @@ -90,7 +88,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): has "LoadModule ssl_module..." before parsing the VirtualHost configuration that was created by Certbot """ - super(CentOSConfigurator, self)._deploy_cert(*args, **kwargs) + super()._deploy_cert(*args, **kwargs) if self.version < (2, 4, 0): self._deploy_loadmodule_ssl_if_needed() @@ -102,9 +100,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator): loadmods = self.parser.find_dir("LoadModule", "ssl_module", exclude=False) - correct_ifmods = [] # type: List[str] - loadmod_args = [] # type: List[str] - loadmod_paths = [] # type: List[str] + correct_ifmods: List[str] = [] + loadmod_args: List[str] = [] + loadmod_paths: List[str] = [] for m in loadmods: noarg_path = m.rpartition("/")[0] path_args = self.parser.get_all_args(noarg_path) @@ -118,8 +116,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator): else: loadmod_args = path_args - if self.parser.not_modssl_ifmodule(noarg_path): # pylint: disable=no-member - if self.parser.loc["default"] in noarg_path: + centos_parser: CentOSParser = cast(CentOSParser, self.parser) + if centos_parser.not_modssl_ifmodule(noarg_path): + if centos_parser.loc["default"] in noarg_path: # LoadModule already in the main configuration file if ("ifmodule/" in noarg_path.lower() or "ifmodule[1]" in noarg_path.lower()): @@ -167,12 +166,12 @@ class CentOSParser(parser.ApacheParser): def __init__(self, *args, **kwargs): # CentOS specific configuration file for Apache self.sysconfig_filep = "/etc/sysconfig/httpd" - super(CentOSParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ # Opportunistic, works if SELinux not enforced - super(CentOSParser, self).update_runtime_variables() + super().update_runtime_variables() self.parse_sysconfig_var() def parse_sysconfig_var(self): diff --git a/certbot-apache/certbot_apache/_internal/override_darwin.py b/certbot-apache/certbot_apache/_internal/override_darwin.py index 106f5fbab..e1dca7f5e 100644 --- a/certbot-apache/certbot_apache/_internal/override_darwin.py +++ b/certbot-apache/certbot_apache/_internal/override_darwin.py @@ -3,26 +3,19 @@ import zope.interface from certbot import interfaces from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class DarwinConfigurator(configurator.ApacheConfigurator): """macOS specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( - server_root="/etc/apache2", + OS_DEFAULTS = OsOptions( vhost_root="/etc/apache2/other", vhost_files="*.conf", - logs_root="/var/log/apache2", ctl="apachectl", version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/apache2/other", - bin=None, ) diff --git a/certbot-apache/certbot_apache/_internal/override_debian.py b/certbot-apache/certbot_apache/_internal/override_debian.py index 9f938046b..7f12f4bbc 100644 --- a/certbot-apache/certbot_apache/_internal/override_debian.py +++ b/certbot-apache/certbot_apache/_internal/override_debian.py @@ -10,6 +10,7 @@ from certbot.compat import filesystem from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions logger = logging.getLogger(__name__) @@ -18,22 +19,11 @@ logger = logging.getLogger(__name__) class DebianConfigurator(configurator.ApacheConfigurator): """Debian specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/sites-available", - vhost_files="*", - logs_root="/var/log/apache2", - ctl="apache2ctl", - version_cmd=['apache2ctl', '-v'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], + OS_DEFAULTS = OsOptions( enmod="a2enmod", dismod="a2dismod", - le_vhost_ext="-le-ssl.conf", handle_modules=True, handle_sites=True, - challenge_location="/etc/apache2", - bin=None, ) def enable_site(self, vhost): @@ -58,7 +48,7 @@ class DebianConfigurator(configurator.ApacheConfigurator): if not os.path.isdir(os.path.dirname(enabled_path)): # For some reason, sites-enabled / sites-available do not exist # Call the parent method - return super(DebianConfigurator, self).enable_site(vhost) + return super().enable_site(vhost) self.reverter.register_file_creation(False, enabled_path) try: os.symlink(vhost.filep, enabled_path) @@ -132,11 +122,11 @@ class DebianConfigurator(configurator.ApacheConfigurator): # Generate reversal command. # Try to be safe here... check that we can probably reverse before # applying enmod command - if not util.exe_exists(self.option("dismod")): + if not util.exe_exists(self.options.dismod): raise errors.MisconfigurationError( "Unable to find a2dismod, please make sure a2enmod and " "a2dismod are configured correctly for certbot.") self.reverter.register_undo_command( - temp, [self.option("dismod"), "-f", mod_name]) - util.run_script([self.option("enmod"), mod_name]) + temp, [self.options.dismod, "-f", mod_name]) + util.run_script([self.options.enmod, mod_name]) diff --git a/certbot-apache/certbot_apache/_internal/override_fedora.py b/certbot-apache/certbot_apache/_internal/override_fedora.py index 9b521846c..3b947a823 100644 --- a/certbot-apache/certbot_apache/_internal/override_fedora.py +++ b/certbot-apache/certbot_apache/_internal/override_fedora.py @@ -7,13 +7,14 @@ from certbot import util from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class FedoraConfigurator(configurator.ApacheConfigurator): """Fedora 29+ specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/httpd", vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", @@ -23,13 +24,7 @@ class FedoraConfigurator(configurator.ApacheConfigurator): restart_cmd=['apachectl', 'graceful'], restart_cmd_alt=['apachectl', 'restart'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/httpd/conf.d", - bin=None, ) def config_test(self): @@ -40,14 +35,14 @@ class FedoraConfigurator(configurator.ApacheConfigurator): during the first (re)start of httpd. """ try: - super(FedoraConfigurator, self).config_test() + super().config_test() except errors.MisconfigurationError: self._try_restart_fedora() def get_parser(self): """Initializes the ApacheParser""" return FedoraParser( - self.option("server_root"), self.option("vhost_root"), + self.options.server_root, self.options.vhost_root, self.version, configurator=self) def _try_restart_fedora(self): @@ -60,7 +55,7 @@ class FedoraConfigurator(configurator.ApacheConfigurator): raise errors.MisconfigurationError(str(err)) # Finish with actual config check to see if systemctl restart helped - super(FedoraConfigurator, self).config_test() + super().config_test() def _prepare_options(self): """ @@ -68,10 +63,12 @@ class FedoraConfigurator(configurator.ApacheConfigurator): instead of httpd and so take advantages of this new bash script in newer versions of Fedora to restart httpd. """ - super(FedoraConfigurator, self)._prepare_options() - self.options["restart_cmd"][0] = 'apachectl' - self.options["restart_cmd_alt"][0] = 'apachectl' - self.options["conftest_cmd"][0] = 'apachectl' + super()._prepare_options() + self.options.restart_cmd[0] = 'apachectl' + if not self.options.restart_cmd_alt: # pragma: no cover + raise ValueError("OS option restart_cmd_alt must be set for Fedora.") + self.options.restart_cmd_alt[0] = 'apachectl' + self.options.conftest_cmd[0] = 'apachectl' class FedoraParser(parser.ApacheParser): @@ -79,12 +76,12 @@ class FedoraParser(parser.ApacheParser): def __init__(self, *args, **kwargs): # Fedora 29+ specific configuration file for Apache self.sysconfig_filep = "/etc/sysconfig/httpd" - super(FedoraParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ # Opportunistic, works if SELinux not enforced - super(FedoraParser, self).update_runtime_variables() + super().update_runtime_variables() self._parse_sysconfig_var() def _parse_sysconfig_var(self): diff --git a/certbot-apache/certbot_apache/_internal/override_gentoo.py b/certbot-apache/certbot_apache/_internal/override_gentoo.py index 59a3d981f..1b86c925e 100644 --- a/certbot-apache/certbot_apache/_internal/override_gentoo.py +++ b/certbot-apache/certbot_apache/_internal/override_gentoo.py @@ -5,29 +5,19 @@ from certbot import interfaces from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class GentooConfigurator(configurator.ApacheConfigurator): """Gentoo specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/apache2", vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", - logs_root="/var/log/apache2", - ctl="apache2ctl", - version_cmd=['apache2ctl', '-v'], - restart_cmd=['apache2ctl', 'graceful'], restart_cmd_alt=['apache2ctl', 'restart'], - conftest_cmd=['apache2ctl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - bin=None, ) def _prepare_options(self): @@ -35,13 +25,15 @@ class GentooConfigurator(configurator.ApacheConfigurator): Override the options dictionary initialization in order to support alternative restart cmd used in Gentoo. """ - super(GentooConfigurator, self)._prepare_options() - self.options["restart_cmd_alt"][0] = self.option("ctl") + super()._prepare_options() + if not self.options.restart_cmd_alt: # pragma: no cover + raise ValueError("OS option restart_cmd_alt must be set for Gentoo.") + self.options.restart_cmd_alt[0] = self.options.ctl def get_parser(self): """Initializes the ApacheParser""" return GentooParser( - self.option("server_root"), self.option("vhost_root"), + self.options.server_root, self.options.vhost_root, self.version, configurator=self) @@ -50,7 +42,7 @@ class GentooParser(parser.ApacheParser): def __init__(self, *args, **kwargs): # Gentoo specific configuration file for Apache2 self.apacheconfig_filep = "/etc/conf.d/apache2" - super(GentooParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ @@ -66,7 +58,7 @@ class GentooParser(parser.ApacheParser): def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.option("ctl"), "modules"] + mod_cmd = [self.configurator.options.ctl, "modules"] matches = apache_util.parse_from_subprocess(mod_cmd, r"(.*)_module") for mod in matches: self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/_internal/override_suse.py b/certbot-apache/certbot_apache/_internal/override_suse.py index afce98dfa..d692fd239 100644 --- a/certbot-apache/certbot_apache/_internal/override_suse.py +++ b/certbot-apache/certbot_apache/_internal/override_suse.py @@ -3,26 +3,21 @@ import zope.interface from certbot import interfaces from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class OpenSUSEConfigurator(configurator.ApacheConfigurator): """OpenSUSE specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( - server_root="/etc/apache2", + OS_DEFAULTS = OsOptions( vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", - logs_root="/var/log/apache2", ctl="apachectl", version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], enmod="a2enmod", dismod="a2dismod", - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - bin=None, ) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 75be0833f..141991ccc 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -3,20 +3,24 @@ import copy import fnmatch import logging import re -import sys +from typing import Dict +from typing import List +from typing import Optional - -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import constants +try: + from augeas import Augeas +except ImportError: # pragma: no cover + Augeas = None # type: ignore + logger = logging.getLogger(__name__) -class ApacheParser(object): +class ApacheParser: """Class handles the fine details of parsing the Apache Configuration. .. todo:: Make parsing general... remove sites-available etc... @@ -41,8 +45,7 @@ class ApacheParser(object): self.configurator = configurator # Initialize augeas - self.aug = None - self.init_augeas() + self.aug = init_augeas() if not self.check_aug_version(): raise errors.NotSupportedError( @@ -50,9 +53,9 @@ class ApacheParser(object): "version 1.2.0 or higher, please make sure you have you have " "those installed.") - self.modules = {} # type: Dict[str, str] - self.parser_paths = {} # type: Dict[str, List[str]] - self.variables = {} # type: Dict[str, str] + self.modules: Dict[str, Optional[str]] = {} + self.parser_paths: Dict[str, List[str]] = {} + self.variables: Dict[str, str] = {} # Find configuration root and make sure augeas can parse it. self.root = os.path.abspath(root) @@ -78,30 +81,13 @@ class ApacheParser(object): # Must also attempt to parse additional virtual host root if vhostroot: self.parse_file(os.path.abspath(vhostroot) + "/" + - self.configurator.option("vhost_files")) + self.configurator.options.vhost_files) # check to see if there were unparsed define statements if version < (2, 4): if self.find_dir("Define", exclude=False): raise errors.PluginError("Error parsing runtime variables") - def init_augeas(self): - """ Initialize the actual Augeas instance """ - - try: - import augeas - except ImportError: # pragma: no cover - raise errors.NoInstallationError("Problem in Augeas installation") - - self.aug = augeas.Augeas( - # specify a directory to load our preferred lens from - loadpath=constants.AUGEAS_LENS_DIR, - # Do not save backup (we do it ourselves), do not load - # anything by default - flags=(augeas.Augeas.NONE | - augeas.Augeas.NO_MODL_AUTOLOAD | - augeas.Augeas.ENABLE_SPAN)) - def check_parsing_errors(self, lens): """Verify Augeas can parse all of the lens files. @@ -265,7 +251,7 @@ class ApacheParser(object): the iteration issue. Else... parse and enable mods at same time. """ - mods = {} # type: Dict[str, str] + mods: Dict[str, str] = {} matches = self.find_dir("LoadModule") iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration @@ -296,7 +282,7 @@ class ApacheParser(object): def update_defines(self): """Updates the dictionary of known variables in the configuration""" - self.variables = apache_util.parse_defines(self.configurator.option("ctl")) + self.variables = apache_util.parse_defines(self.configurator.options.ctl) def update_includes(self): """Get includes from httpd process, and add them to DOM if needed""" @@ -306,7 +292,7 @@ class ApacheParser(object): # configuration files _ = self.find_dir("Include") - matches = apache_util.parse_includes(self.configurator.option("ctl")) + matches = apache_util.parse_includes(self.configurator.options.ctl) if matches: for i in matches: if not self.parsed_in_current(i): @@ -315,7 +301,7 @@ class ApacheParser(object): def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - matches = apache_util.parse_modules(self.configurator.option("ctl")) + matches = apache_util.parse_modules(self.configurator.options.ctl) for mod in matches: self.add_mod(mod.strip()) @@ -552,7 +538,7 @@ class ApacheParser(object): else: arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg) - ordered_matches = [] # type: List[str] + ordered_matches: List[str] = [] # TODO: Wildcards should be included in alphabetical order # https://httpd.apache.org/docs/2.4/mod/core.html#include @@ -737,9 +723,6 @@ class ApacheParser(object): :rtype: str """ - if sys.version_info < (3, 6): - # This strips off final /Z(?ms) - return fnmatch.translate(clean_fn_match)[:-7] # pragma: no cover # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z return fnmatch.translate(clean_fn_match)[4:-3] # pragma: no cover @@ -954,3 +937,19 @@ def get_aug_path(file_path): """ return "/files%s" % file_path + + +def init_augeas() -> Augeas: + """ Initialize the actual Augeas instance """ + + if not Augeas: # pragma: no cover + raise errors.NoInstallationError("Problem in Augeas installation") + + return Augeas( + # specify a directory to load our preferred lens from + loadpath=constants.AUGEAS_LENS_DIR, + # Do not save backup (we do it ourselves), do not load + # anything by default + flags=(Augeas.NONE | + Augeas.NO_MODL_AUTOLOAD | + Augeas.ENABLE_SPAN)) diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f129343b3..898e4e3e7 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -24,7 +24,7 @@ setup( description="Apache plugin for Certbot", url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index abe72a5d0..85a17ab31 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -1,4 +1,6 @@ """Tests for AugeasParserNode classes""" +from typing import List + try: import mock except ImportError: # pragma: no cover @@ -27,7 +29,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- """Test AugeasParserNode using available test configurations""" def setUp(self): # pylint: disable=arguments-differ - super(AugeasParserNodeTest, self).setUp() + super().setUp() with mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.get_parsernode_root") as mock_parsernode: mock_parsernode.side_effect = _get_augeasnode_mock( @@ -107,7 +109,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- def test_set_parameters(self): servernames = self.config.parser_root.find_directives("servername") - names = [] # type: List[str] + names: List[str] = [] for servername in servernames: names += servername.parameters self.assertFalse("going_to_set_this" in names) diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index db1c46b52..8c8ba4873 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -18,7 +18,7 @@ class AutoHSTSTest(util.ApacheTest): # pylint: disable=protected-access def setUp(self): # pylint: disable=arguments-differ - super(AutoHSTSTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) diff --git a/certbot-apache/tests/centos6_test.py b/certbot-apache/tests/centos6_test.py index 27b4f8e80..bfbe22ad0 100644 --- a/certbot-apache/tests/centos6_test.py +++ b/certbot-apache/tests/centos6_test.py @@ -36,9 +36,9 @@ class CentOS6Tests(util.ApacheTest): test_dir = "centos6_apache/apache" config_root = "centos6_apache/apache/httpd" vhost_root = "centos6_apache/apache/httpd/conf.d" - super(CentOS6Tests, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, diff --git a/certbot-apache/tests/centos_test.py b/certbot-apache/tests/centos_test.py index b7e9c1cb6..a92c37979 100644 --- a/certbot-apache/tests/centos_test.py +++ b/certbot-apache/tests/centos_test.py @@ -41,9 +41,9 @@ class FedoraRestartTest(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(FedoraRestartTest, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="fedora_old") @@ -96,9 +96,9 @@ class MultipleVhostsTestCentOS(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(MultipleVhostsTestCentOS, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, diff --git a/certbot-apache/tests/complex_parsing_test.py b/certbot-apache/tests/complex_parsing_test.py index 8b795b0b6..e36bd85d1 100644 --- a/certbot-apache/tests/complex_parsing_test.py +++ b/certbot-apache/tests/complex_parsing_test.py @@ -11,7 +11,7 @@ class ComplexParserTest(util.ParserTest): """Apache Parser Test.""" def setUp(self): # pylint: disable=arguments-differ - super(ComplexParserTest, self).setUp( + super().setUp( "complex_parsing", "complex_parsing") self.setup_variables() diff --git a/certbot-apache/tests/configurator_reverter_test.py b/certbot-apache/tests/configurator_reverter_test.py index 8596195d8..d8f5ddd05 100644 --- a/certbot-apache/tests/configurator_reverter_test.py +++ b/certbot-apache/tests/configurator_reverter_test.py @@ -16,7 +16,7 @@ class ConfiguratorReverterTest(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ - super(ConfiguratorReverterTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 302bf0023..8620955aa 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -30,7 +30,7 @@ class MultipleVhostsTest(util.ApacheTest): """Test two standard well-configured HTTP vhosts.""" def setUp(self): # pylint: disable=arguments-differ - super(MultipleVhostsTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) @@ -103,9 +103,9 @@ class MultipleVhostsTest(util.ApacheTest): "handle_modules", "handle_sites", "ctl"] exp = {} - for k in ApacheConfigurator.OS_DEFAULTS: + for k in ApacheConfigurator.OS_DEFAULTS.__dict__.keys(): if k in parserargs: - exp[k.replace("_", "-")] = ApacheConfigurator.OS_DEFAULTS[k] + exp[k.replace("_", "-")] = getattr(ApacheConfigurator.OS_DEFAULTS, k) # Special cases exp["vhost-root"] = None @@ -128,14 +128,13 @@ class MultipleVhostsTest(util.ApacheTest): def test_all_configurators_defaults_defined(self): from certbot_apache._internal.entrypoint import OVERRIDE_CLASSES from certbot_apache._internal.configurator import ApacheConfigurator - parameters = set(ApacheConfigurator.OS_DEFAULTS.keys()) + parameters = set(ApacheConfigurator.OS_DEFAULTS.__dict__.keys()) for cls in OVERRIDE_CLASSES.values(): - self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.keys()))) + self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.__dict__.keys()))) def test_constant(self): self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in - self.config.option("server_root")) - self.assertEqual(self.config.option("nonexistent"), None) + self.config.options.server_root) @certbot_util.patch_get_utility() def test_get_all_names(self, mock_getutility): @@ -1477,9 +1476,9 @@ class AugeasVhostsTest(util.ApacheTest): td = "debian_apache_2_4/augeas_vhosts" cr = "debian_apache_2_4/augeas_vhosts/apache2" vr = "debian_apache_2_4/augeas_vhosts/apache2/sites-available" - super(AugeasVhostsTest, self).setUp(test_dir=td, - config_root=cr, - vhost_root=vr) + super().setUp(test_dir=td, + config_root=cr, + vhost_root=vr) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, @@ -1556,9 +1555,9 @@ class MultiVhostsTest(util.ApacheTest): td = "debian_apache_2_4/multi_vhosts" cr = "debian_apache_2_4/multi_vhosts/apache2" vr = "debian_apache_2_4/multi_vhosts/apache2/sites-available" - super(MultiVhostsTest, self).setUp(test_dir=td, - config_root=cr, - vhost_root=vr) + super().setUp(test_dir=td, + config_root=cr, + vhost_root=vr) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, @@ -1661,7 +1660,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" def setUp(self): # pylint: disable=arguments-differ - super(InstallSslOptionsConfTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) @@ -1774,7 +1773,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): # ssl_module statically linked self.config._openssl_version = None self.config.parser.modules['ssl_module'] = None - self.config.options['bin'] = '/fake/path/to/httpd' + self.config.options.bin = '/fake/path/to/httpd' with mock.patch("certbot_apache._internal.configurator." "ApacheConfigurator._open_module_file") as mock_omf: mock_omf.return_value = some_string_contents @@ -1810,7 +1809,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): # When ssl_module is statically linked but --apache-bin not provided self.config._openssl_version = None - self.config.options['bin'] = None + self.config.options.bin = None self.config.parser.modules['ssl_module'] = None with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: self.assertEqual(self.config.openssl_version(), None) diff --git a/certbot-apache/tests/debian_test.py b/certbot-apache/tests/debian_test.py index 192e3cba5..c72b8b6ae 100644 --- a/certbot-apache/tests/debian_test.py +++ b/certbot-apache/tests/debian_test.py @@ -20,7 +20,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): _multiprocess_can_split_ = True def setUp(self): # pylint: disable=arguments-differ - super(MultipleVhostsTestDebian, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="debian") diff --git a/certbot-apache/tests/fedora_test.py b/certbot-apache/tests/fedora_test.py index 50831802b..d48559cee 100644 --- a/certbot-apache/tests/fedora_test.py +++ b/certbot-apache/tests/fedora_test.py @@ -46,9 +46,9 @@ class FedoraRestartTest(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(FedoraRestartTest, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="fedora") @@ -90,9 +90,9 @@ class MultipleVhostsTestFedora(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(MultipleVhostsTestFedora, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, diff --git a/certbot-apache/tests/gentoo_test.py b/certbot-apache/tests/gentoo_test.py index 64f7d1062..cf39ff5cb 100644 --- a/certbot-apache/tests/gentoo_test.py +++ b/certbot-apache/tests/gentoo_test.py @@ -50,9 +50,9 @@ class MultipleVhostsTestGentoo(util.ApacheTest): test_dir = "gentoo_apache/apache" config_root = "gentoo_apache/apache/apache2" vhost_root = "gentoo_apache/apache/apache2/vhosts.d" - super(MultipleVhostsTestGentoo, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) # pylint: disable=line-too-long with mock.patch("certbot_apache._internal.override_gentoo.GentooParser.update_runtime_variables"): diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 696cd4a54..71f2db500 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -1,6 +1,7 @@ """Test for certbot_apache._internal.http_01.""" import unittest import errno +from typing import List try: import mock @@ -23,10 +24,10 @@ class ApacheHttp01Test(util.ApacheTest): """Test for certbot_apache._internal.http_01.ApacheHttp01.""" def setUp(self, *args, **kwargs): # pylint: disable=arguments-differ - super(ApacheHttp01Test, self).setUp(*args, **kwargs) + super().setUp(*args, **kwargs) self.account_key = self.rsa512jwk - self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] + self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = [] vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") # Takes the vhosts for encryption-example.demo, certbot.demo diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py index 7aedec31d..e30eca153 100644 --- a/certbot-apache/tests/parser_test.py +++ b/certbot-apache/tests/parser_test.py @@ -16,7 +16,7 @@ class BasicParserTest(util.ParserTest): """Apache Parser Test.""" def setUp(self): # pylint: disable=arguments-differ - super(BasicParserTest, self).setUp() + super().setUp() def tearDown(self): shutil.rmtree(self.temp_dir) @@ -305,11 +305,9 @@ class BasicParserTest(util.ParserTest): self.assertRaises( errors.PluginError, self.parser.update_runtime_variables) - @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.option") @mock.patch("certbot_apache._internal.apache_util.subprocess.Popen") - def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_opt): + def test_update_runtime_vars_bad_ctl(self, mock_popen): mock_popen.side_effect = OSError - mock_opt.return_value = "nonexistent" self.assertRaises( errors.MisconfigurationError, self.parser.update_runtime_variables) @@ -332,14 +330,14 @@ class BasicParserTest(util.ParserTest): class ParserInitTest(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ - super(ParserInitTest, self).setUp() + super().setUp() def tearDown(self): shutil.rmtree(self.temp_dir) shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) - @mock.patch("certbot_apache._internal.parser.ApacheParser.init_augeas") + @mock.patch("certbot_apache._internal.parser.init_augeas") def test_prepare_no_augeas(self, mock_init_augeas): from certbot_apache._internal.parser import ApacheParser mock_init_augeas.side_effect = errors.NoInstallationError diff --git a/certbot-apache/tests/parsernode_configurator_test.py b/certbot-apache/tests/parsernode_configurator_test.py index 7fbec2540..411871a43 100644 --- a/certbot-apache/tests/parsernode_configurator_test.py +++ b/certbot-apache/tests/parsernode_configurator_test.py @@ -20,7 +20,7 @@ class ConfiguratorParserNodeTest(util.ApacheTest): # pylint: disable=too-many-p """Test AugeasParserNode using available test configurations""" def setUp(self): # pylint: disable=arguments-differ - super(ConfiguratorParserNodeTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, diff --git a/certbot-apache/tests/parsernode_test.py b/certbot-apache/tests/parsernode_test.py index a86952f53..4ea5f8415 100644 --- a/certbot-apache/tests/parsernode_test.py +++ b/certbot-apache/tests/parsernode_test.py @@ -18,7 +18,7 @@ class DummyParserNode(interfaces.ParserNode): self.dirty = dirty self.filepath = filepath self.metadata = metadata - super(DummyParserNode, self).__init__(**kwargs) + super().__init__(**kwargs) def save(self, msg): # pragma: no cover """Save""" @@ -38,7 +38,7 @@ class DummyCommentNode(DummyParserNode): """ comment, kwargs = util.commentnode_kwargs(kwargs) self.comment = comment - super(DummyCommentNode, self).__init__(**kwargs) + super().__init__(**kwargs) class DummyDirectiveNode(DummyParserNode): @@ -54,7 +54,7 @@ class DummyDirectiveNode(DummyParserNode): self.parameters = parameters self.enabled = enabled - super(DummyDirectiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) def set_parameters(self, parameters): # pragma: no cover """Set parameters""" diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index 18c7e5aca..a0b44d188 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -67,7 +67,7 @@ class ParserTest(ApacheTest): def setUp(self, test_dir="debian_apache_2_4/multiple_vhosts", config_root="debian_apache_2_4/multiple_vhosts/apache2", vhost_root="debian_apache_2_4/multiple_vhosts/apache2/sites-available"): - super(ParserTest, self).setUp(test_dir, config_root, vhost_root) + super().setUp(test_dir, config_root, vhost_root) zope.component.provideUtility(display_util.FileDisplay(sys.stdout, False)) @@ -123,11 +123,11 @@ def get_apache_configurator( version=version, use_parsernode=use_parsernode, openssl_version=openssl_version) if not conf_vhost_path: - config_class.OS_DEFAULTS["vhost_root"] = vhost_path + config_class.OS_DEFAULTS.vhost_root = vhost_path else: # Custom virtualhost path was requested config.config.apache_vhost_root = conf_vhost_path - config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"] + config.config.apache_ctl = config_class.OS_DEFAULTS.ctl config.prepare() return config diff --git a/certbot-auto b/certbot-auto index 002fd5ffc..c37c45596 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.12.0" +LE_AUTO_VERSION="1.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -800,12 +800,14 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -1487,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.14.0 \ + --hash=sha256:67b4d26ceaea6c7f8325d0d45169e7a165a2cabc7122c84bc971ba068ca19cca \ + --hash=sha256:959ea90c6bb8dca38eab9772722cb940972ef6afcd5f15deef08b3c3636841eb +acme==1.14.0 \ + --hash=sha256:4f48c41261202f1a389ec2986b2580b58f53e0d5a1ae2463b34318d78b87fc66 \ + --hash=sha256:61daccfb0343628cbbca551a7fc4c82482113952c21db3fe0c585b7c98fa1c35 +certbot-apache==1.14.0 \ + --hash=sha256:b757038db23db707c44630fecb46e99172bd791f0db5a8e623c0842613c4d3d9 \ + --hash=sha256:887fe4a21af2de1e5c2c9428bacba6eb7c1219257bc70f1a1d8447c8a321adb0 +certbot-nginx==1.14.0 \ + --hash=sha256:8916a815437988d6c192df9f035bb7a176eab20eee0956677b335d0698d243fb \ + --hash=sha256:cc2a8a0de56d9bb6b2efbda6c80c647dad8db2bb90675cac03ade94bd5fc8597 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-ci/certbot_integration_tests/assets/hook.py b/certbot-ci/certbot_integration_tests/assets/hook.py index 483892a93..c11704f47 100755 --- a/certbot-ci/certbot_integration_tests/assets/hook.py +++ b/certbot-ci/certbot_integration_tests/assets/hook.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function import os import sys diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index b9854b402..9af1bc15c 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -7,14 +7,14 @@ import tempfile from certbot_integration_tests.utils import certbot_call -class IntegrationTestsContext(object): +class IntegrationTestsContext: """General fixture describing a certbot integration tests context""" def __init__(self, request): self.request = request - if hasattr(request.config, 'slaveinput'): # Worker node - self.worker_id = request.config.slaveinput['slaveid'] - acme_xdist = request.config.slaveinput['acme_xdist'] + if hasattr(request.config, 'workerinput'): # Worker node + self.worker_id = request.config.workerinput['workerid'] + acme_xdist = request.config.workerinput['acme_xdist'] else: # Primary node self.worker_id = 'primary' acme_xdist = request.config.acme_xdist diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 2d3d93669..c9fa5ff65 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -1,5 +1,4 @@ """Module executing integration tests against certbot core.""" -from __future__ import print_function import os from os.path import exists @@ -9,19 +8,20 @@ import shutil import subprocess import time -from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1, SECP384R1, SECP521R1 +from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1 +from cryptography.hazmat.primitives.asymmetric.ec import SECP384R1 +from cryptography.hazmat.primitives.asymmetric.ec import SECP521R1 from cryptography.x509 import NameOID - import pytest from certbot_integration_tests.certbot_tests import context as certbot_context from certbot_integration_tests.certbot_tests.assertions import assert_cert_count_for_lineage from certbot_integration_tests.certbot_tests.assertions import assert_elliptic_key -from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_owner from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_permissions from certbot_integration_tests.certbot_tests.assertions import assert_equals_world_read_permissions from certbot_integration_tests.certbot_tests.assertions import assert_hook_execution +from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key from certbot_integration_tests.certbot_tests.assertions import assert_saved_renew_hook from certbot_integration_tests.certbot_tests.assertions import assert_world_no_permissions from certbot_integration_tests.certbot_tests.assertions import assert_world_read_permissions diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index 230fb0eda..5e6ed5562 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -6,7 +6,6 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ -from __future__ import print_function import contextlib import subprocess import sys @@ -35,7 +34,7 @@ def pytest_configure(config): Standard pytest hook used to add a configuration logic for each node of a pytest run. :param config: the current pytest configuration """ - if not hasattr(config, 'slaveinput'): # If true, this is the primary node + if not hasattr(config, 'workerinput'): # If true, this is the primary node with _print_on_err(): _setup_primary_node(config) @@ -45,8 +44,8 @@ def pytest_configure_node(node): Standard pytest-xdist hook used to configure a worker node. :param node: current worker node """ - node.slaveinput['acme_xdist'] = node.config.acme_xdist - node.slaveinput['dns_xdist'] = node.config.dns_xdist + node.workerinput['acme_xdist'] = node.config.acme_xdist + node.workerinput['dns_xdist'] = node.config.dns_xdist @contextlib.contextmanager diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/context.py b/certbot-ci/certbot_integration_tests/nginx_tests/context.py index 6f0f833a0..3ee1766a1 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/context.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/context.py @@ -11,7 +11,7 @@ from certbot_integration_tests.utils import misc class IntegrationTestsContext(certbot_context.IntegrationTestsContext): """General fixture describing a certbot-nginx integration tests context""" def __init__(self, request): - super(IntegrationTestsContext, self).__init__(request) + super().__init__(request) self.nginx_root = os.path.join(self.workspace, 'nginx') os.mkdir(self.nginx_root) @@ -29,7 +29,7 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): def cleanup(self): self._stop_nginx() - super(IntegrationTestsContext, self).cleanup() + super().cleanup() def certbot_test_nginx(self, args): """ diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py index 8a2d48a50..2c52c8523 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -1,8 +1,8 @@ """Module executing integration tests against certbot with nginx plugin.""" import os import ssl - from typing import List + import pytest from certbot_integration_tests.nginx_tests import context as nginx_context @@ -32,8 +32,8 @@ def test_context(request): '--preferred-challenges', 'http' ], {'default_server': False}), ], indirect=['context']) -def test_certificate_deployment(certname_pattern, params, context): - # type: (str, List[str], nginx_context.IntegrationTestsContext) -> None +def test_certificate_deployment(certname_pattern: str, params: List[str], + context: nginx_context.IntegrationTestsContext) -> None: """ Test various scenarios to deploy a certificate to nginx using certbot. """ diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py index bdedee1fe..3d4147313 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -1,7 +1,7 @@ """Module to handle the context of RFC2136 integration tests.""" -import tempfile from contextlib import contextmanager +import tempfile from pkg_resources import resource_filename from pytest import skip @@ -13,13 +13,12 @@ from certbot_integration_tests.utils import certbot_call class IntegrationTestsContext(certbot_context.IntegrationTestsContext): """Integration test context for certbot-dns-rfc2136""" def __init__(self, request): - super(IntegrationTestsContext, self).__init__(request) + super().__init__(request) self.request = request - self._dns_xdist = None - if hasattr(request.config, 'slaveinput'): # Worker node - self._dns_xdist = request.config.slaveinput['dns_xdist'] + if hasattr(request.config, 'workerinput'): # Worker node + self._dns_xdist = request.config.workerinput['dns_xdist'] else: # Primary node self._dns_xdist = request.config.dns_xdist @@ -45,7 +44,6 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): src_file = resource_filename('certbot_integration_tests', 'assets/bind-config/rfc2136-credentials-{}.ini.tpl' .format(label)) - contents = None with open(src_file, 'r') as f: contents = f.read().format( diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index bbbdd196b..ceebbe7ed 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -1,6 +1,5 @@ #!/usr/bin/env python """Module to setup an ACME CA server environment able to run multiple tests in parallel""" -from __future__ import print_function import argparse import errno @@ -12,18 +11,18 @@ import subprocess import sys import tempfile import time - from typing import List + import requests +# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils import misc from certbot_integration_tests.utils import pebble_artifacts from certbot_integration_tests.utils import proxy -# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils.constants import * -class ACMEServer(object): +class ACMEServer: """ ACMEServer configures and handles the lifecycle of an ACME CA server and an HTTP reverse proxy instance, to allow parallel execution of integration tests against the unique http-01 port @@ -52,7 +51,7 @@ class ACMEServer(object): self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' self._proxy = http_proxy self._workspace = tempfile.mkdtemp() - self._processes = [] # type: List[subprocess.Popen] + self._processes: List[subprocess.Popen] = [] self._stdout = sys.stdout if stdout else open(os.devnull, 'w') self._dns_server = dns_server self._http_01_port = http_01_port diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py index c9e46cdc7..965ab6881 100755 --- a/certbot-ci/certbot_integration_tests/utils/certbot_call.py +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -1,11 +1,10 @@ #!/usr/bin/env python """Module to call certbot in test mode""" -from __future__ import absolute_import +from distutils.version import LooseVersion import os import subprocess import sys -from distutils.version import LooseVersion import certbot_integration_tests # pylint: disable=wildcard-import,unused-wildcard-import diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index 416f6567e..48f5a533e 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -1,7 +1,5 @@ #!/usr/bin/env python """Module to setup an RFC2136-capable DNS server""" -from __future__ import print_function - import os import os.path import shutil @@ -10,6 +8,7 @@ import subprocess import sys import tempfile import time +from typing import Optional from pkg_resources import resource_filename @@ -21,7 +20,7 @@ BIND_BIND_ADDRESS = ("127.0.0.1", 45953) BIND_TEST_QUERY = bytearray.fromhex("0011cb37000000010000000000000000010003") -class DNSServer(object): +class DNSServer: """ DNSServer configures and handles the lifetime of an RFC2136-capable server. DNServer provides access to the dns_xdist parameter, listing the address and port @@ -40,7 +39,7 @@ class DNSServer(object): self.bind_root = tempfile.mkdtemp() - self.process = None # type: subprocess.Popen + self.process: Optional[subprocess.Popen] = None self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]} @@ -113,8 +112,7 @@ class DNSServer(object): self.stop() raise - def _wait_until_ready(self, attempts=30): - # type: (int) -> None + def _wait_until_ready(self, attempts: int = 30) -> None: """ Polls the DNS server over TCP until it gets a response, or until it runs out of attempts and raises a ValueError. @@ -122,6 +120,9 @@ class DNSServer(object): but otherwise the contents are ignored. :param int attempts: The number of attempts to make. """ + if not self.process: + raise ValueError("DNS server has not been started. Please run start() first.") + for _ in range(attempts): if self.process.poll(): raise ValueError("BIND9 server stopped unexpectedly") diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index e6b9f0c88..2fac494f2 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -26,8 +26,8 @@ from OpenSSL import crypto import pkg_resources import requests -from certbot_integration_tests.utils.constants import \ - PEBBLE_ALTERNATE_ROOTS, PEBBLE_MANAGEMENT_URL +from certbot_integration_tests.utils.constants import PEBBLE_ALTERNATE_ROOTS +from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL RSA_KEY_TYPE = 'rsa' ECDSA_KEY_TYPE = 'ecdsa' @@ -311,7 +311,7 @@ def echo(keyword, path=None): if not re.match(r'^\w+$', keyword): raise ValueError('Error, keyword `{0}` is not a single keyword.' .format(keyword)) - return '{0} -c "from __future__ import print_function; print(\'{1}\')"{2}'.format( + return '{0} -c "print(\'{1}\')"{2}'.format( os.path.basename(sys.executable), keyword, ' >> "{0}"'.format(path) if path else '') diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index cd62e1a7f..918a5fd04 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -7,7 +7,8 @@ import stat import pkg_resources import requests -from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT, MOCK_OCSP_SERVER_PORT +from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT +from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT PEBBLE_VERSION = 'v2.3.0' ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py index 4db72998f..3458929ad 100755 --- a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py @@ -29,10 +29,7 @@ class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediates/0', verify=False) issuer_cert = x509.load_pem_x509_certificate(request.content, default_backend()) - try: - content_len = int(self.headers.getheader('content-length', 0)) - except AttributeError: - content_len = int(self.headers.get('Content-Length')) + content_len = int(self.headers.get('Content-Length')) ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len)) response = requests.get('{0}/cert-status-by-serial/{1}'.format( diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index 9f9c1f462..9d52b6268 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -7,6 +7,13 @@ from setuptools import setup version = '0.32.0.dev0' +# setuptools 36.2+ is needed for support for environment markers +min_setuptools_version='36.2' +# This conditional isn't necessary, but it provides better error messages to +# people who try to install this package with older versions of setuptools. +if LooseVersion(setuptools_version) < LooseVersion(min_setuptools_version): + raise RuntimeError(f'setuptools {min_setuptools_version}+ is required') + install_requires = [ 'coverage', 'cryptography', @@ -14,30 +21,24 @@ install_requires = [ 'pyopenssl', 'pytest', 'pytest-cov', - 'pytest-xdist', + # This version is needed for "worker" attributes we currently use like + # "workerinput". See https://github.com/pytest-dev/pytest-xdist/pull/268. + 'pytest-xdist>=1.22.1', 'python-dateutil', + # This dependency needs to be added using environment markers to avoid its + # installation on Linux. + 'pywin32>=300 ; sys_platform == "win32"', 'pyyaml', 'requests', ] -# Add pywin32 on Windows platforms to handle low-level system calls. -# This dependency needs to be added using environment markers to avoid its installation on Linux. -# However environment markers are supported only with setuptools >= 36.2. -# So this dependency is not added for old Linux distributions with old setuptools, -# in order to allow these systems to build certbot from sources. -if LooseVersion(setuptools_version) >= LooseVersion('36.2'): - install_requires.append("pywin32>=224 ; sys_platform == 'win32'") -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') - setup( name='certbot-ci', version=version, description="Certbot continuous integration framework", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-ci/snap_integration_tests/dns_tests/test_main.py b/certbot-ci/snap_integration_tests/dns_tests/test_main.py index 016355334..d008efc67 100644 --- a/certbot-ci/snap_integration_tests/dns_tests/test_main.py +++ b/certbot-ci/snap_integration_tests/dns_tests/test_main.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -import pytest -import subprocess import glob import os import re +import subprocess + +import pytest @pytest.fixture(autouse=True, scope="module") @@ -43,4 +44,4 @@ def test_dns_plugin_install(dns_snap_path): 'certbot:certbot-metadata']) subprocess.check_call(['snap', 'install', '--dangerous', dns_snap_path]) finally: - subprocess.call(['snap', 'remove', 'plugin_name']) + subprocess.call(['snap', 'remove', plugin_name]) diff --git a/certbot-ci/windows_installer_integration_tests/conftest.py b/certbot-ci/windows_installer_integration_tests/conftest.py index c6a89c323..8a9de057f 100644 --- a/certbot-ci/windows_installer_integration_tests/conftest.py +++ b/certbot-ci/windows_installer_integration_tests/conftest.py @@ -6,7 +6,7 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ -from __future__ import print_function + import os ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) diff --git a/certbot-ci/windows_installer_integration_tests/test_main.py b/certbot-ci/windows_installer_integration_tests/test_main.py index c8c347aa8..f699b736a 100644 --- a/certbot-ci/windows_installer_integration_tests/test_main.py +++ b/certbot-ci/windows_installer_integration_tests/test_main.py @@ -1,8 +1,8 @@ import os +import re +import subprocess import time import unittest -import subprocess -import re @unittest.skipIf(os.name != 'nt', reason='Windows installer tests must be run on Windows.') diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index 2b3f94581..909a9d37c 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -22,7 +22,7 @@ class Proxy(configurators_common.Proxy): def __init__(self, args): """Initializes the plugin with the given command line args""" - super(Proxy, self).__init__(args) + super().__init__(args) self.le_config.apache_le_vhost_ext = "-le-ssl.conf" self.modules = self.server_root = self.test_conf = self.version = None @@ -34,7 +34,7 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" - config = super(Proxy, self).load_config() + config = super().load_config() self._all_names, self._test_names = _get_names(config) server_root = _get_server_root(config) @@ -54,9 +54,9 @@ class Proxy(configurators_common.Proxy): def _prepare_configurator(self): """Prepares the Apache plugin for testing""" - for k in entrypoint.ENTRYPOINT.OS_DEFAULTS: + for k in entrypoint.ENTRYPOINT.OS_DEFAULTS.__dict__.keys(): setattr(self.le_config, "apache_" + k, - entrypoint.ENTRYPOINT.OS_DEFAULTS[k]) + getattr(entrypoint.ENTRYPOINT.OS_DEFAULTS, k)) self._configurator = entrypoint.ENTRYPOINT( config=configuration.NamespaceConfig(self.le_config), @@ -65,7 +65,7 @@ class Proxy(configurators_common.Proxy): def cleanup_from_tests(self): """Performs any necessary cleanup from running plugin tests""" - super(Proxy, self).cleanup_from_tests() + super().cleanup_from_tests() mock.patch.stopall() diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index 34aa9133f..bf26291ea 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -11,7 +11,7 @@ from certbot_compatibility_test import util logger = logging.getLogger(__name__) -class Proxy(object): +class Proxy: """A common base for compatibility test configurators""" @classmethod @@ -33,7 +33,9 @@ class Proxy(object): self.args = args self.http_port = 80 self.https_port = 443 - self._configurator = self._all_names = self._test_names = None + self._configurator = None + self._all_names = None + self._test_names = None def __getattr__(self, name): """Wraps the configurator methods""" @@ -93,5 +95,7 @@ class Proxy(object): """Installs cert""" cert_path, key_path, chain_path = self.copy_certs_and_keys( cert_path, key_path, chain_path) + if not self._configurator: + raise ValueError("Configurator plugin is not set.") self._configurator.deploy_cert( domain, cert_path, key_path, chain_path, fullchain_path) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py index 7cb4e9722..e209480e3 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py @@ -2,10 +2,10 @@ import os import shutil import subprocess +from typing import Set import zope.interface -from acme.magic_typing import Set from certbot._internal import configuration from certbot_compatibility_test import errors from certbot_compatibility_test import interfaces @@ -21,7 +21,7 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" - config = super(Proxy, self).load_config() + config = super().load_config() self._all_names, self._test_names = _get_names(config) server_root = _get_server_root(config) @@ -68,7 +68,7 @@ def _get_server_root(config): def _get_names(config): """Returns all and testable domain names in config""" - all_names = set() # type: Set[str] + all_names: Set[str] = set() for root, _dirs, files in os.walk(config): for this_file in files: update_names = _get_server_names(root, this_file) diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index f11b9fdf8..4bb16e63f 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -8,6 +8,8 @@ import shutil import sys import tempfile import time +from typing import List +from typing import Tuple import OpenSSL from urllib3.util import connection @@ -15,8 +17,6 @@ from urllib3.util import connection from acme import challenges from acme import crypto_util from acme import messages -from acme.magic_typing import List -from acme.magic_typing import Tuple from certbot import achallenges from certbot import errors as le_errors from certbot.tests import acme_util @@ -178,7 +178,7 @@ def test_enhancements(plugin, domains): "enhancements") return False - domains_and_info = [(domain, []) for domain in domains] # type: List[Tuple[str, List[bool]]] + domains_and_info: List[Tuple[str, List[bool]]] = [(domain, []) for domain in domains] for domain, info in domains_and_info: try: diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index 9a082523a..226b8585b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -10,7 +10,7 @@ from acme import errors as acme_errors logger = logging.getLogger(__name__) -class Validator(object): +class Validator: """Collection of functions to test a live webserver's configuration""" def certificate(self, cert, name, alt_host=None, port=443): diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index af19b126e..9567bbef6 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' install_requires = [ 'certbot', @@ -24,7 +24,7 @@ setup( description="Compatibility tests for Certbot", url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index 5978af85c..8f42b3ce9 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -1,16 +1,17 @@ """DNS Authenticator for Cloudflare.""" import logging +from typing import Any +from typing import Dict +from typing import List +from typing import Optional import CloudFlare import zope.interface -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import List - from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -30,12 +31,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 120 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add) + super().add_parser_arguments(add) add('credentials', help='Cloudflare credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -80,12 +81,14 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_cloudflare_client().del_txt_record(domain, validation_name, validation) def _get_cloudflare_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") if self.credentials.conf('api-token'): return _CloudflareClient(None, self.credentials.conf('api-token')) return _CloudflareClient(self.credentials.conf('email'), self.credentials.conf('api-key')) -class _CloudflareClient(object): +class _CloudflareClient: """ Encapsulates all communication with the Cloudflare API. """ @@ -173,7 +176,7 @@ class _CloudflareClient(object): """ zone_name_guesses = dns_common.base_domain_name_guesses(domain) - zones = [] # type: List[Dict[str, Any]] + zones: List[Dict[str, Any]] = [] code = msg = None for zone_name in zone_name_guesses: diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index eab6cdb70..3e3a3d1ba 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="Cloudflare DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py index e9adf4ed9..d7075a84d 100644 --- a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py +++ b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py @@ -27,7 +27,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def setUp(self): from certbot_dns_cloudflare._internal.dns_cloudflare import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write({"cloudflare_email": EMAIL, "cloudflare_api_key": API_KEY}, path) diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py index 654c04c70..1ec5cb5ab 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py @@ -1,5 +1,6 @@ """DNS Authenticator for CloudXNS DNS.""" import logging +from typing import Optional from lexicon.providers import cloudxns import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='CloudXNS credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -56,6 +58,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_cloudxns_client().del_txt_record(domain, validation_name, validation) def _get_cloudxns_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _CloudXNSLexiconClient(self.credentials.conf('api-key'), self.credentials.conf('secret-key'), self.ttl) @@ -67,7 +71,7 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, secret_key, ttl): - super(_CloudXNSLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('cloudxns', { 'ttl': ttl, diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 83513ef7c..20b499327 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="CloudXNS DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py index 43c69790f..81dea5ca4 100644 --- a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py +++ b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py @@ -26,7 +26,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_cloudxns._internal.dns_cloudxns import Authenticator diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index e0c9561a2..23b669847 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -1,5 +1,6 @@ """DNS Authenticator for DigitalOcean.""" import logging +from typing import Optional import digitalocean import zope.interface @@ -7,6 +8,7 @@ import zope.interface from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -21,14 +23,15 @@ class Authenticator(dns_common.DNSAuthenticator): description = 'Obtain certificates using a DNS TXT record (if you are ' + \ 'using DigitalOcean for DNS).' + ttl = 30 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add) + super().add_parser_arguments(add) add('credentials', help='DigitalOcean credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -45,16 +48,19 @@ class Authenticator(dns_common.DNSAuthenticator): ) def _perform(self, domain, validation_name, validation): - self._get_digitalocean_client().add_txt_record(domain, validation_name, validation) + self._get_digitalocean_client().add_txt_record(domain, validation_name, validation, + self.ttl) def _cleanup(self, domain, validation_name, validation): self._get_digitalocean_client().del_txt_record(domain, validation_name, validation) def _get_digitalocean_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _DigitalOceanClient(self.credentials.conf('token')) -class _DigitalOceanClient(object): +class _DigitalOceanClient: """ Encapsulates all communication with the DigitalOcean API. """ @@ -62,13 +68,15 @@ class _DigitalOceanClient(object): def __init__(self, token): self.manager = digitalocean.Manager(token=token) - def add_txt_record(self, domain_name, record_name, record_content): + def add_txt_record(self, domain_name: str, record_name: str, record_content: str, + record_ttl: int): """ Add a TXT record using the supplied information. :param str domain_name: The domain to use to associate the record with. :param str record_name: The record name (typically beginning with '_acme-challenge.'). :param str record_content: The record content (typically the challenge validation). + :param int record_ttl: The record TTL. :raises certbot.errors.PluginError: if an error occurs communicating with the DigitalOcean API """ @@ -89,7 +97,8 @@ class _DigitalOceanClient(object): result = domain.create_new_domain_record( type='TXT', name=self._compute_record_name(domain, record_name), - data=record_content) + data=record_content, + ttl=record_ttl) # ttl kwarg is only effective starting python-digitalocean 1.15.0 record_id = result['domain_record']['id'] @@ -99,7 +108,7 @@ class _DigitalOceanClient(object): raise errors.PluginError('Error adding TXT record using the DigitalOcean API: {0}' .format(e)) - def del_txt_record(self, domain_name, record_name, record_content): + def del_txt_record(self, domain_name: str, record_name: str, record_content: str): """ Delete a TXT record using the supplied information. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 6e60444cf..fba2fbc5f 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,12 +4,12 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'python-digitalocean>=1.11', + 'python-digitalocean>=1.11', # 1.15.0 or newer is recommended for TTL support 'setuptools>=39.0.1', 'zope.interface', ] @@ -36,7 +36,7 @@ setup( description="DigitalOcean DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py index a752f52d0..6088262ee 100644 --- a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py @@ -23,7 +23,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def setUp(self): from certbot_dns_digitalocean._internal.dns_digitalocean import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write({"digitalocean_token": TOKEN}, path) @@ -40,7 +40,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def test_perform(self): self.auth.perform([self.achall]) - expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] + expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY, 30)] self.assertEqual(expected, self.mock_client.mock_calls) def test_cleanup(self): @@ -58,6 +58,7 @@ class DigitalOceanClientTest(unittest.TestCase): record_prefix = "_acme-challenge" record_name = record_prefix + "." + DOMAIN record_content = "bar" + record_ttl = 60 def setUp(self): from certbot_dns_digitalocean._internal.dns_digitalocean import _DigitalOceanClient @@ -78,25 +79,27 @@ class DigitalOceanClientTest(unittest.TestCase): self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock] - self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content) + self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content, + self.record_ttl) domain_mock.create_new_domain_record.assert_called_with(type='TXT', name=self.record_prefix, - data=self.record_content) + data=self.record_content, + ttl=self.record_ttl) def test_add_txt_record_fail_to_find_domain(self): self.manager.get_all_domains.return_value = [] self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_add_txt_record_error_finding_domain(self): self.manager.get_all_domains.side_effect = API_ERROR self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_add_txt_record_error_creating_record(self): domain_mock = mock.MagicMock() @@ -107,7 +110,7 @@ class DigitalOceanClientTest(unittest.TestCase): self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_del_txt_record(self): first_record_mock = mock.MagicMock() diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py index 9f7f100d7..858ee8925 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py @@ -1,5 +1,6 @@ """DNS Authenticator for DNSimple DNS.""" import logging +from typing import Optional from lexicon.providers import dnsimple import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='DNSimple credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -54,6 +56,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_dnsimple_client().del_txt_record(domain, validation_name, validation) def _get_dnsimple_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _DNSimpleLexiconClient(self.credentials.conf('token'), self.ttl) @@ -63,7 +67,7 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, token, ttl): - super(_DNSimpleLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('dnssimple', { 'ttl': ttl, diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index f1fcfd11d..c0da63d71 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -47,7 +47,7 @@ setup( description="DNSimple DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py index 40eba4754..fc3dc5b1f 100644 --- a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py +++ b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py @@ -20,7 +20,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_dnsimple._internal.dns_dnsimple import Authenticator diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py index 4a1fcffc3..67903e19d 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py @@ -1,5 +1,6 @@ """DNS Authenticator for DNS Made Easy DNS.""" import logging +from typing import Optional from lexicon.providers import dnsmadeeasy import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) + super().add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help='DNS Made Easy credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -58,6 +60,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_dnsmadeeasy_client().del_txt_record(domain, validation_name, validation) def _get_dnsmadeeasy_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _DNSMadeEasyLexiconClient(self.credentials.conf('api-key'), self.credentials.conf('secret-key'), self.ttl) @@ -69,7 +73,7 @@ class _DNSMadeEasyLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, secret_key, ttl): - super(_DNSMadeEasyLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('dnsmadeeasy', { 'ttl': ttl, diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 185048a2d..2feae0bd1 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="DNS Made Easy DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py index 4a69e977c..a04716d95 100644 --- a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py +++ b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py @@ -22,7 +22,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_dnsmadeeasy._internal.dns_dnsmadeeasy import Authenticator diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py index 39deddae5..57ff01671 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py @@ -1,12 +1,15 @@ """DNS Authenticator for Gehirn Infrastructure Service DNS.""" import logging +from typing import Optional from lexicon.providers import gehirn import zope.interface +from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='Gehirn Infrastructure Service credentials file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -57,6 +60,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_gehirn_client().del_txt_record(domain, validation_name, validation) def _get_gehirn_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _GehirnLexiconClient( self.credentials.conf('api-token'), self.credentials.conf('api-secret'), @@ -70,7 +75,7 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_token, api_secret, ttl): - super(_GehirnLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('gehirn', { 'ttl': ttl, @@ -84,4 +89,4 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return None # Expected errors when zone name guess is wrong - return super(_GehirnLexiconClient, self)._handle_http_error(e, domain_name) + return super()._handle_http_error(e, domain_name) diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 0ae9c1bf7..d24f6b309 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -35,7 +35,7 @@ setup( description="Gehirn Infrastructure Service DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-gehirn/tests/dns_gehirn_test.py b/certbot-dns-gehirn/tests/dns_gehirn_test.py index 0598a5eb5..1310f74ca 100644 --- a/certbot-dns-gehirn/tests/dns_gehirn_test.py +++ b/certbot-dns-gehirn/tests/dns_gehirn_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_gehirn._internal.dns_gehirn import Authenticator diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py index 2d448c590..67ed34a45 100644 --- a/certbot-dns-google/certbot_dns_google/__init__.py +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -51,8 +51,16 @@ are automatically obtained by certbot through the `metadata service :caption: Example credentials file: { - "type": "service_account", - ... + "type": "service_account", + "project_id": "...", + "private_key_id": "...", + "private_key": "...", + "client_email": "...", + "client_id": "...", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "..." } The path to this file can be provided interactively or using the diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index 4b0d91463..3a2686a63 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -32,13 +32,9 @@ class Authenticator(dns_common.DNSAuthenticator): 'for DNS).') ttl = 60 - def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None - @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) + super().add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help=('Path to Google Cloud DNS service account JSON file. (See {0} for' + 'information about creating a service account and {1} for information about the' + @@ -76,7 +72,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _GoogleClient(self.conf('credentials')) -class _GoogleClient(object): +class _GoogleClient: """ Encapsulates all communication with the Google Cloud DNS API. """ diff --git a/certbot-dns-google/docs/_ext/jsonlexer.py b/certbot-dns-google/docs/_ext/jsonlexer.py deleted file mode 100644 index 1ad004d2b..000000000 --- a/certbot-dns-google/docs/_ext/jsonlexer.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Copied from https://stackoverflow.com/a/16863232""" - -def setup(app): - # enable Pygments json lexer - try: - import pygments - if pygments.__version__ >= '1.5': - # use JSON lexer included in recent versions of Pygments - from pygments.lexers import JsonLexer - else: - # use JSON lexer from pygments-json if installed - from pygson.json_lexer import JSONLexer as JsonLexer - except ImportError: - pass # not fatal if we have old (or no) Pygments and no pygments-json - else: - app.add_lexer('json', JsonLexer()) diff --git a/certbot-dns-google/docs/conf.py b/certbot-dns-google/docs/conf.py index 06bb99f46..f4c1f661e 100644 --- a/certbot-dns-google/docs/conf.py +++ b/certbot-dns-google/docs/conf.py @@ -35,8 +35,7 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', - 'jsonlexer'] + 'sphinx.ext.viewcode'] autodoc_member_order = 'bysource' autodoc_default_flags = ['show-inheritance'] diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index b16d014c6..dd43f4992 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -39,7 +39,7 @@ setup( description="Google Cloud DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 396a6c8bd..aa3ff35b5 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -26,14 +26,14 @@ PROJECT_ID = "test-test-1" class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_google._internal.dns_google import Authenticator path = os.path.join(self.tempdir, 'file.json') open(path, "wb").close() - super(AuthenticatorTest, self).setUp() + super().setUp() self.config = mock.MagicMock(google_credentials=path, google_propagation_seconds=0) # don't wait during tests @@ -401,7 +401,7 @@ class GoogleClientTest(unittest.TestCase): self.assertRaises(ServerNotFoundError, _GoogleClient.get_project_id) -class DummyResponse(object): +class DummyResponse: """ Dummy object to create a fake HTTPResponse (the actual one requires a socket and we only need the status attribute) diff --git a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py index c1b5e066f..b1649cf61 100644 --- a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py @@ -1,6 +1,7 @@ """DNS Authenticator for Linode.""" import logging import re +from typing import Optional from lexicon.providers import linode from lexicon.providers import linode4 @@ -10,6 +11,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): description = 'Obtain certificates using a DNS TXT record (if you are using Linode for DNS).' def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=120) + super().add_parser_arguments(add, default_propagation_seconds=120) add('credentials', help='Linode credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -56,6 +58,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_linode_client().del_txt_record(domain, validation_name, validation) def _get_linode_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") api_key = self.credentials.conf('key') api_version = self.credentials.conf('version') if api_version == '': @@ -81,7 +85,7 @@ class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, api_version): - super(_LinodeLexiconClient, self).__init__() + super().__init__() self.api_version = api_version diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 21ccf9d42..72c7ee2fd 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -35,7 +35,7 @@ setup( description="Linode DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-linode/tests/dns_linode_test.py b/certbot-dns-linode/tests/dns_linode_test.py index fb9b1aa93..9861d2ab0 100644 --- a/certbot-dns-linode/tests/dns_linode_test.py +++ b/certbot-dns-linode/tests/dns_linode_test.py @@ -22,7 +22,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write({"linode_key": TOKEN}, path) diff --git a/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py b/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py index d5b499c72..ed90b63d9 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py +++ b/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py @@ -1,5 +1,6 @@ """DNS Authenticator for LuaDNS DNS.""" import logging +from typing import Optional from lexicon.providers import luadns import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='LuaDNS credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -55,6 +57,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_luadns_client().del_txt_record(domain, validation_name, validation) def _get_luadns_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _LuaDNSLexiconClient(self.credentials.conf('email'), self.credentials.conf('token'), self.ttl) @@ -66,7 +70,7 @@ class _LuaDNSLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, email, token, ttl): - super(_LuaDNSLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('luadns', { 'ttl': ttl, diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 2312d6fcc..b2c54779f 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="LuaDNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-luadns/tests/dns_luadns_test.py b/certbot-dns-luadns/tests/dns_luadns_test.py index a1242582f..7592e2323 100644 --- a/certbot-dns-luadns/tests/dns_luadns_test.py +++ b/certbot-dns-luadns/tests/dns_luadns_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_luadns._internal.dns_luadns import Authenticator diff --git a/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py b/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py index d328d80ce..ce46ad835 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py +++ b/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py @@ -1,5 +1,6 @@ """DNS Authenticator for NS1 DNS.""" import logging +from typing import Optional from lexicon.providers import nsone import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='NS1 credentials file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -54,6 +56,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_nsone_client().del_txt_record(domain, validation_name, validation) def _get_nsone_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _NS1LexiconClient(self.credentials.conf('api-key'), self.ttl) @@ -63,7 +67,7 @@ class _NS1LexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, ttl): - super(_NS1LexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('nsone', { 'ttl': ttl, diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 658027b9a..a78b6c3b7 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="NS1 DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-nsone/tests/dns_nsone_test.py b/certbot-dns-nsone/tests/dns_nsone_test.py index 83371252f..3754f9811 100644 --- a/certbot-dns-nsone/tests/dns_nsone_test.py +++ b/certbot-dns-nsone/tests/dns_nsone_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_nsone._internal.dns_nsone import Authenticator diff --git a/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py index 11ae6b8f0..54fd6d791 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py +++ b/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py @@ -1,5 +1,6 @@ """DNS Authenticator for OVH DNS.""" import logging +from typing import Optional from lexicon.providers import ovh import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='OVH credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -60,6 +62,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_ovh_client().del_txt_record(domain, validation_name, validation) def _get_ovh_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _OVHLexiconClient( self.credentials.conf('endpoint'), self.credentials.conf('application-key'), @@ -75,7 +79,7 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, endpoint, application_key, application_secret, consumer_key, ttl): - super(_OVHLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('ovh', { 'ttl': ttl, @@ -102,4 +106,4 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): if domain_name in str(e) and str(e).endswith('not found'): return - super(_OVHLexiconClient, self)._handle_general_error(e, domain_name) + super()._handle_general_error(e, domain_name) diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index b4f73ddb4..4831fa480 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="OVH DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-ovh/tests/dns_ovh_test.py b/certbot-dns-ovh/tests/dns_ovh_test.py index dd0f3058b..7f93967eb 100644 --- a/certbot-dns-ovh/tests/dns_ovh_test.py +++ b/certbot-dns-ovh/tests/dns_ovh_test.py @@ -23,7 +23,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_ovh._internal.dns_ovh import Authenticator diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py index 57e9506f2..28fd27eec 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py @@ -1,5 +1,6 @@ """DNS Authenticator using RFC 2136 Dynamic Updates.""" import logging +from typing import Optional import dns.flags import dns.message @@ -15,6 +16,7 @@ import zope.interface from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -43,12 +45,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 120 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) + super().add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help='RFC 2136 credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -80,6 +82,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_rfc2136_client().del_txt_record(validation_name, validation) def _get_rfc2136_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _RFC2136Client(self.credentials.conf('server'), int(self.credentials.conf('port') or self.PORT), self.credentials.conf('name'), @@ -88,7 +92,7 @@ class Authenticator(dns_common.DNSAuthenticator): dns.tsig.HMAC_MD5)) -class _RFC2136Client(object): +class _RFC2136Client: """ Encapsulates all communication with the target DNS server. """ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index ce74611cd..a19753e79 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="RFC 2136 DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py index dc4a73a04..c2b80defe 100644 --- a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py @@ -28,7 +28,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def setUp(self): from certbot_dns_rfc2136._internal.dns_rfc2136 import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write(VALID_CONFIG, path) diff --git a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py index 6250d2274..9e470bdde 100644 --- a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py @@ -2,15 +2,15 @@ import collections import logging import time +from typing import DefaultDict +from typing import Dict +from typing import List import boto3 from botocore.exceptions import ClientError from botocore.exceptions import NoCredentialsError import zope.interface -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot.plugins import dns_common @@ -37,9 +37,9 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 10 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.r53 = boto3.client("route53") - self._resource_records = collections.defaultdict(list) # type: DefaultDict[str, List[Dict[str, str]]] + self._resource_records: DefaultDict[str, List[Dict[str, str]]] = collections.defaultdict(list) def more_info(self): # pylint: disable=missing-function-docstring return "Solve a DNS01 challenge using AWS Route53" diff --git a/certbot-dns-route53/certbot_dns_route53/authenticator.py b/certbot-dns-route53/certbot_dns_route53/authenticator.py index 2987934a1..060d2fa38 100644 --- a/certbot-dns-route53/certbot_dns_route53/authenticator.py +++ b/certbot-dns-route53/certbot_dns_route53/authenticator.py @@ -18,4 +18,4 @@ class Authenticator(dns_route53.Authenticator): def __init__(self, *args, **kwargs): warnings.warn("The 'authenticator' module was renamed 'dns_route53'", DeprecationWarning) - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8def9a702..b1c1d786c 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="Route53 DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-route53/tests/dns_route53_test.py b/certbot-dns-route53/tests/dns_route53_test.py index 1fd191c69..69b6b115d 100644 --- a/certbot-dns-route53/tests/dns_route53_test.py +++ b/certbot-dns-route53/tests/dns_route53_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(unittest.TestCase, dns_test_common.BaseAuthenticatorTest def setUp(self): from certbot_dns_route53._internal.dns_route53 import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() self.config = mock.MagicMock() diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py index 67cfb2e97..668c5e1f8 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py @@ -1,12 +1,15 @@ """DNS Authenticator for Sakura Cloud DNS.""" import logging +from typing import Optional from lexicon.providers import sakuracloud import zope.interface +from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments( + super().add_parser_arguments( add, default_propagation_seconds=90) add('credentials', help='Sakura Cloud credentials file.') @@ -60,6 +63,8 @@ class Authenticator(dns_common.DNSAuthenticator): domain, validation_name, validation) def _get_sakuracloud_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _SakuraCloudLexiconClient( self.credentials.conf('api-token'), self.credentials.conf('api-secret'), @@ -73,7 +78,7 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_token, api_secret, ttl): - super(_SakuraCloudLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('sakuracloud', { 'ttl': ttl, @@ -87,4 +92,4 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return None # Expected errors when zone name guess is wrong - return super(_SakuraCloudLexiconClient, self)._handle_http_error(e, domain_name) + return super()._handle_http_error(e, domain_name) diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 6f4f8e506..7b76bb324 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -35,7 +35,7 @@ setup( description="Sakura Cloud DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py index af94336b3..1c64df372 100644 --- a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py +++ b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_sakuracloud._internal.dns_sakuracloud import Authenticator diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 15fbe61f7..9fb81efaa 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-lines """Nginx Configuration""" from distutils.version import LooseVersion import logging @@ -6,6 +7,12 @@ import socket import subprocess import tempfile import time +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Text +from typing import Tuple import OpenSSL import pkg_resources @@ -13,10 +20,6 @@ import zope.interface from acme import challenges from acme import crypto_util as acme_crypto_util -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Text from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -99,24 +102,23 @@ class NginxConfigurator(common.Installer): """ version = kwargs.pop("version", None) openssl_version = kwargs.pop("openssl_version", None) - super(NginxConfigurator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) # Files to save self.save_notes = "" # For creating new vhosts if no names match - self.new_vhost = None + self.new_vhost: Optional[obj.VirtualHost] = None # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] - self._wildcard_redirect_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {} + self._wildcard_redirect_vhosts: Dict[str, List[obj.VirtualHost]] = {} # Add number of outstanding challenges self._chall_out = 0 # These will be set in the prepare function - self.parser = None self.version = version self.openssl_version = openssl_version self._enhance_func = {"redirect": self._enable_redirect, @@ -124,6 +126,7 @@ class NginxConfigurator(common.Installer): "staple-ocsp": self._enable_ocsp_stapling} self.reverter.recovery_routine() + self.parser: parser.NginxParser @property def mod_ssl_conf_src(self): @@ -377,10 +380,13 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain, allow_port_mismatch, port): + def _vhost_from_duplicated_default(self, domain: str, allow_port_mismatch: bool, port: str + ) -> obj.VirtualHost: """if allow_port_mismatch is False, only server blocks with matching ports will be used as a default server block template. """ + assert self.parser is not None # prepare should already have been called here + if self.new_vhost is None: default_vhost = self._get_default_vhost(domain, allow_port_mismatch, port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, @@ -509,7 +515,7 @@ class NginxConfigurator(common.Installer): match['rank'] += NO_SSL_MODIFIER return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): + def choose_redirect_vhosts(self, target_name: str, port: str) -> List[obj.VirtualHost]: """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -523,9 +529,6 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param str port: port number - :param bool create_if_no_match: If we should create a new vhost from default - when there is no match found. If we can't choose a default, raise a - MisconfigurationError. :returns: vhosts associated with name :rtype: list of :class:`~certbot_nginx._internal.obj.VirtualHost` @@ -538,32 +541,75 @@ class NginxConfigurator(common.Installer): else: matches = self._get_redirect_ranked_matches(target_name, port) vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] - if not vhosts and create_if_no_match: - vhosts = [self._vhost_from_duplicated_default(target_name, False, port)] return vhosts - def _port_matches(self, test_port, matching_port): + def choose_auth_vhosts(self, target_name: str) -> Tuple[List[obj.VirtualHost], + List[obj.VirtualHost]]: + """Returns a list of HTTP and HTTPS vhosts with a server_name matching target_name. + + If no HTTP vhost exists, one will be cloned from the default vhost. If that fails, no HTTP + vhost will be returned. + + :param str target_name: non-wildcard domain name + + :returns: tuple of HTTP and HTTPS virtualhosts + :rtype: tuple of :class:`~certbot_nginx._internal.obj.VirtualHost` + + """ + vhosts = [m['vhost'] for m in self._get_ranked_matches(target_name) if m and 'vhost' in m] + http_vhosts = [vh for vh in vhosts if + self._vhost_listening(vh, str(self.config.http01_port), False)] + https_vhosts = [vh for vh in vhosts if + self._vhost_listening(vh, str(self.config.https_port), True)] + + # If no HTTP vhost matches, try create one from the default_server on http01_port. + if not http_vhosts: + try: + http_vhosts = [self._vhost_from_duplicated_default(target_name, False, + str(self.config.http01_port))] + except errors.MisconfigurationError: + http_vhosts = [] + + return http_vhosts, https_vhosts + + def _port_matches(self, test_port: str, matching_port: str) -> bool: # test_port is a number, matching is a number or "" or None if matching_port == "" or matching_port is None: # if no port is specified, Nginx defaults to listening on port 80. return test_port == self.DEFAULT_LISTEN_PORT return test_port == matching_port - def _vhost_listening_on_port_no_ssl(self, vhost, port): - found_matching_port = False - if not vhost.addrs: - # if there are no listen directives at all, Nginx defaults to - # listening on port 80. - found_matching_port = (port == self.DEFAULT_LISTEN_PORT) - else: - for addr in vhost.addrs: - if self._port_matches(port, addr.get_port()) and not addr.ssl: - found_matching_port = True + def _vhost_listening(self, vhost: obj.VirtualHost, port: str, ssl: bool) -> bool: + """Tests whether a vhost has an address listening on a port with SSL enabled or disabled. - if found_matching_port: - # make sure we don't have an 'ssl on' directive - return not self.parser.has_ssl_on_directive(vhost) - return False + :param `obj.VirtualHost` vhost: The vhost whose addresses will be tested + :param port str: The port number as a string that the address should be bound to + :param bool ssl: Whether SSL should be enabled or disabled on the address + + :returns: Whether the vhost has an address listening on the port and protocol. + :rtype: bool + + """ + assert self.parser is not None # prepare should already have been called here + + # if the 'ssl on' directive is present on the vhost, all its addresses have SSL enabled + all_addrs_are_ssl = self.parser.has_ssl_on_directive(vhost) + + # if we want ssl vhosts: either 'ssl on' or 'addr.ssl' should be enabled + # if we want plaintext vhosts: neither 'ssl on' nor 'addr.ssl' should be enabled + _ssl_matches = lambda addr: addr.ssl or all_addrs_are_ssl if ssl else \ + not addr.ssl and not all_addrs_are_ssl + + # if there are no listen directives at all, Nginx defaults to + # listening on port 80. + if not vhost.addrs: + return port == self.DEFAULT_LISTEN_PORT and ssl == all_addrs_are_ssl + + return any(self._port_matches(port, addr.get_port()) and _ssl_matches(addr) + for addr in vhost.addrs) + + def _vhost_listening_on_port_no_ssl(self, vhost: obj.VirtualHost, port: str) -> bool: + return self._vhost_listening(vhost, port, False) def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -595,7 +641,7 @@ class NginxConfigurator(common.Installer): :rtype: set """ - all_names = set() # type: Set[str] + all_names: Set[str] = set() for vhost in self.parser.get_vhosts(): try: @@ -1064,7 +1110,7 @@ class NginxConfigurator(common.Installer): :raises .errors.PluginError: If unable to recover the configuration """ - super(NginxConfigurator, self).recovery_routine() + super().recovery_routine() self.new_vhost = None self.parser.load() @@ -1087,7 +1133,7 @@ class NginxConfigurator(common.Installer): the function is unable to correctly revert the configuration """ - super(NginxConfigurator, self).rollback_checkpoints(rollback) + super().rollback_checkpoints(rollback) self.new_vhost = None self.parser.load() @@ -1176,7 +1222,7 @@ def nginx_restart(nginx_ctl, nginx_conf, sleep_duration): """ try: - reload_output = u"" # type: Text + reload_output: Text = u"" with tempfile.TemporaryFile() as out: proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"], env=util.env_no_snap_for_external_calls(), diff --git a/certbot-nginx/certbot_nginx/_internal/constants.py b/certbot-nginx/certbot_nginx/_internal/constants.py index 1f058e7ef..52f4d8238 100644 --- a/certbot-nginx/certbot_nginx/_internal/constants.py +++ b/certbot-nginx/certbot_nginx/_internal/constants.py @@ -1,8 +1,7 @@ """nginx plugin constants.""" import platform - -from acme.magic_typing import Any -from acme.magic_typing import Dict +from typing import Any +from typing import Dict FREEBSD_DARWIN_SERVER_ROOT = "/usr/local/etc/nginx" LINUX_SERVER_ROOT = "/etc/nginx" @@ -15,11 +14,11 @@ elif platform.system() in ('NetBSD',): else: server_root_tmp = LINUX_SERVER_ROOT -CLI_DEFAULTS = dict( +CLI_DEFAULTS: Dict[str, Any] = dict( server_root=server_root_tmp, ctl="nginx", sleep_seconds=1 -) # type: Dict[str, Any] +) """CLI defaults.""" diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index 40f994988..716af0898 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -2,9 +2,11 @@ import io import logging +from typing import List +from typing import Optional from acme import challenges -from acme.magic_typing import List +from certbot import achallenges from certbot import errors from certbot.compat import os from certbot.plugins import common @@ -35,7 +37,7 @@ class NginxHttp01(common.ChallengePerformer): """ def __init__(self, configurator): - super(NginxHttp01, self).__init__(configurator) + super().__init__(configurator) self.challenge_conf = os.path.join( configurator.config.config_dir, "le_http_01_cert_challenge.conf") @@ -111,7 +113,7 @@ class NginxHttp01(common.ChallengePerformer): :returns: list of :class:`certbot_nginx._internal.obj.Addr` to apply :rtype: list """ - addresses = [] # type: List[obj.Addr] + addresses: List[obj.Addr] = [] default_addr = "%s" % self.configurator.config.http01_port ipv6_addr = "[::]:{0}".format( self.configurator.config.http01_port) @@ -138,13 +140,12 @@ class NginxHttp01(common.ChallengePerformer): def _get_validation_path(self, achall): return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token")) - def _make_server_block(self, achall): + def _make_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge) -> List: """Creates a server block for a challenge. + :param achall: Annotated HTTP-01 challenge - :type achall: - :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` - :param list addrs: addresses of challenged domain - :class:`list` of type :class:`~nginx.obj.Addr` + :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :returns: server block for the challenge host :rtype: list """ @@ -172,34 +173,35 @@ class NginxHttp01(common.ChallengePerformer): return location_directive - def _make_or_mod_server_block(self, achall): - """Modifies a server block to respond to a challenge. + def _make_or_mod_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge + ) -> Optional[List]: + """Modifies server blocks to respond to a challenge. Returns a new HTTP server block + to add to the configuration if an existing one can't be found. :param achall: Annotated HTTP-01 challenge - :type achall: - :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + + :returns: new server block to be added, if any + :rtype: list """ - try: - vhosts = self.configurator.choose_redirect_vhosts(achall.domain, - '%i' % self.configurator.config.http01_port, create_if_no_match=True) - except errors.MisconfigurationError: + http_vhosts, https_vhosts = self.configurator.choose_auth_vhosts(achall.domain) + + new_vhost: Optional[list] = None + if not http_vhosts: # Couldn't find either a matching name+port server block # or a port+default_server block, so create a dummy block - return self._make_server_block(achall) + new_vhost = self._make_server_block(achall) - # len is max 1 because Nginx doesn't authenticate wildcards - # if len were or vhosts None, we would have errored - vhost = vhosts[0] + # Modify any existing server blocks + for vhost in set(http_vhosts + https_vhosts): + location_directive = [self._location_directive_for_achall(achall)] - # Modify existing server block - location_directive = [self._location_directive_for_achall(achall)] + self.configurator.parser.add_server_directives(vhost, location_directive) - self.configurator.parser.add_server_directives(vhost, - location_directive) + rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', + ' ', '$1', ' ', 'break']] + self.configurator.parser.add_server_directives(vhost, + rewrite_directive, insert_at_top=True) - rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', - ' ', '$1', ' ', 'break']] - self.configurator.parser.add_server_directives(vhost, - rewrite_directive, insert_at_top=True) - return None + return new_vhost diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index f043b0e4d..787f7c363 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -2,6 +2,8 @@ # Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed) import copy import logging +from typing import Any +from typing import IO from pyparsing import Combine from pyparsing import Forward @@ -15,11 +17,11 @@ from pyparsing import restOfLine from pyparsing import stringEnd from pyparsing import White from pyparsing import ZeroOrMore -from acme.magic_typing import IO, Any # pylint: disable=unused-import logger = logging.getLogger(__name__) -class RawNginxParser(object): + +class RawNginxParser: # pylint: disable=pointless-statement """A class that parses nginx configuration with pyparsing.""" @@ -69,7 +71,7 @@ class RawNginxParser(object): """Returns the parsed tree as a list.""" return self.parse().asList() -class RawNginxDumper(object): +class RawNginxDumper: """A class that dumps nginx configuration from the provided tree.""" def __init__(self, blocks): self.blocks = blocks @@ -104,57 +106,9 @@ class RawNginxDumper(object): return ''.join(self) -# Shortcut functions to respect Python's serialization interface -# (like pyyaml, picker or json) - -def loads(source): - """Parses from a string. - - :param str source: The string to parse - :returns: The parsed tree - :rtype: list - - """ - return UnspacedList(RawNginxParser(source).as_list()) - - -def load(_file): - """Parses from a file. - - :param file _file: The file to parse - :returns: The parsed tree - :rtype: list - - """ - return loads(_file.read()) - - -def dumps(blocks): - # type: (UnspacedList) -> str - """Dump to a Unicode string. - - :param UnspacedList block: The parsed tree - :rtype: str - - """ - return str(RawNginxDumper(blocks.spaced)) - - -def dump(blocks, _file): - # type: (UnspacedList, IO[Any]) -> None - """Dump to a file. - - :param UnspacedList block: The parsed tree - :param IO[Any] _file: The file stream to dump to. It must be opened with - Unicode encoding. - :rtype: None - - """ - _file.write(dumps(blocks)) - - spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == '' + class UnspacedList(list): """Wrap a list [of lists], making any whitespace entries magically invisible""" @@ -273,3 +227,50 @@ class UnspacedList(list): idx -= 1 pos += 1 return idx0 + spaces + + +# Shortcut functions to respect Python's serialization interface +# (like pyyaml, picker or json) + +def loads(source): + """Parses from a string. + + :param str source: The string to parse + :returns: The parsed tree + :rtype: list + + """ + return UnspacedList(RawNginxParser(source).as_list()) + + +def load(_file): + """Parses from a file. + + :param file _file: The file to parse + :returns: The parsed tree + :rtype: list + + """ + return loads(_file.read()) + + +def dumps(blocks: UnspacedList) -> str: + """Dump to a Unicode string. + + :param UnspacedList block: The parsed tree + :rtype: six.text_type + + """ + return str(RawNginxDumper(blocks.spaced)) + + +def dump(blocks: UnspacedList, _file: IO[Any]) -> None: + """Dump to a file. + + :param UnspacedList block: The parsed tree + :param IO[Any] _file: The file stream to dump to. It must be opened with + Unicode encoding. + :rtype: None + + """ + _file.write(dumps(blocks)) diff --git a/certbot-nginx/certbot_nginx/_internal/obj.py b/certbot-nginx/certbot_nginx/_internal/obj.py index 2dd02f180..44be0e598 100644 --- a/certbot-nginx/certbot_nginx/_internal/obj.py +++ b/certbot-nginx/certbot_nginx/_internal/obj.py @@ -1,7 +1,6 @@ """Module contains classes used by the Nginx Configurator.""" import re - from certbot.plugins import common ADD_HEADER_DIRECTIVE = 'add_header' @@ -36,7 +35,7 @@ class Addr(common.Addr): CANONICAL_UNSPECIFIED_ADDRESS = UNSPECIFIED_IPV4_ADDRESSES[0] def __init__(self, host, port, ssl, default, ipv6, ipv6only): - super(Addr, self).__init__((host, port)) + super().__init__((host, port)) self.ssl = ssl self.default = default self.ipv6 = ipv6 @@ -121,7 +120,7 @@ class Addr(common.Addr): def __hash__(self): # pylint: disable=useless-super-delegation # Python 3 requires explicit overridden for __hash__ # See certbot-apache/certbot_apache/_internal/obj.py for more information - return super(Addr, self).__hash__() + return super().__hash__() def super_eq(self, other): """Check ip/port equality, with IPv6 support. @@ -133,7 +132,7 @@ class Addr(common.Addr): self.tup[1]), self.ipv6) == \ common.Addr((other.CANONICAL_UNSPECIFIED_ADDRESS, other.tup[1]), other.ipv6) - return super(Addr, self).__eq__(other) + return super().__eq__(other) def __eq__(self, other): if isinstance(other, self.__class__): @@ -143,7 +142,7 @@ class Addr(common.Addr): return False -class VirtualHost(object): +class VirtualHost: """Represents an Nginx Virtualhost. :ivar str filep: file path of VH diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index fe5d7bb31..28833b1f7 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -5,23 +5,25 @@ import glob import io import logging import re +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Tuple +from typing import Union import pyparsing -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Tuple -from acme.magic_typing import Union from certbot import errors from certbot.compat import os from certbot_nginx._internal import nginxparser from certbot_nginx._internal import obj +from certbot_nginx._internal.nginxparser import UnspacedList logger = logging.getLogger(__name__) -class NginxParser(object): +class NginxParser: """Class handles the fine details of parsing the Nginx Configuration. :ivar str root: Normalized absolute path to the server root @@ -31,7 +33,7 @@ class NginxParser(object): """ def __init__(self, root): - self.parsed = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] + self.parsed: Dict[str, Union[List, nginxparser.UnspacedList]] = {} self.root = os.path.abspath(root) self.config_root = self._find_config_root() @@ -93,7 +95,7 @@ class NginxParser(object): """ servers = self._get_raw_servers() - addr_to_ssl = {} # type: Dict[Tuple[str, str], bool] + addr_to_ssl: Dict[Tuple[str, str], bool] = {} for filename in servers: for server, _ in servers[filename]: # Parse the server block to save addr info @@ -105,12 +107,11 @@ class NginxParser(object): addr_to_ssl[addr_tuple] = addr.ssl or addr_to_ssl[addr_tuple] return addr_to_ssl - def _get_raw_servers(self): + def _get_raw_servers(self) -> Dict: # pylint: disable=cell-var-from-loop - # type: () -> Dict """Get a map of unparsed all server blocks """ - servers = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] + servers: Dict[str, Union[List, nginxparser.UnspacedList]] = {} for filename in self.parsed: tree = self.parsed[filename] servers[filename] = [] @@ -243,6 +244,8 @@ class NginxParser(object): tree = self.parsed[filename] if ext: filename = filename + os.path.extsep + ext + if not isinstance(tree, UnspacedList): + raise ValueError(f"Error tree {tree} is not an UnspacedList") try: if lazy and not tree.is_dirty(): continue @@ -361,8 +364,9 @@ class NginxParser(object): except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) - def duplicate_vhost(self, vhost_template, remove_singleton_listen_params=False, - only_directives=None): + def duplicate_vhost(self, vhost_template: obj.VirtualHost, + remove_singleton_listen_params: bool = False, + only_directives: Optional[List] = None) -> obj.VirtualHost: """Duplicate the vhost in the configuration files. :param :class:`~certbot_nginx._internal.obj.VirtualHost` vhost_template: The vhost @@ -739,9 +743,9 @@ def _parse_server_raw(server): :rtype: dict """ - addrs = set() # type: Set[obj.Addr] - ssl = False # type: bool - names = set() # type: Set[str] + addrs: Set[obj.Addr] = set() + ssl: bool = False + names: Set[str] = set() apply_ssl_to_all_addrs = False diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index d616a1a99..d4d332c47 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -1,11 +1,12 @@ +# type: ignore +# This module is not used for now, so we just skip type check for the sake of simplicity. """ This file contains parsing routines and object classes to help derive meaning from raw lists of tokens from pyparsing. """ import abc import logging +from typing import List - -from acme.magic_typing import List from certbot import errors logger = logging.getLogger(__name__) @@ -13,7 +14,7 @@ COMMENT = " managed by Certbot" COMMENT_BLOCK = ["#", COMMENT] -class Parsable(object): +class Parsable: """ Abstract base class for "Parsable" objects whose underlying representation is a tree of lists. @@ -23,7 +24,7 @@ class Parsable(object): __metaclass__ = abc.ABCMeta def __init__(self, parent=None): - self._data = [] # type: List[object] + self._data: List[object] = [] self._tabs = None self.parent = parent @@ -121,7 +122,7 @@ class Statements(Parsable): precede any more statements. """ def __init__(self, parent=None): - super(Statements, self).__init__(parent) + super().__init__(parent) self._trailing_whitespace = None # ======== Begin overridden functions @@ -166,7 +167,7 @@ class Statements(Parsable): def dump(self, include_spaces=False): """ Dumps this object by first dumping each statement, then appending its trailing whitespace (if `include_spaces` is set) """ - data = super(Statements, self).dump(include_spaces) + data = super().dump(include_spaces) if include_spaces and self._trailing_whitespace is not None: return data + [self._trailing_whitespace] return data @@ -182,7 +183,7 @@ class Statements(Parsable): def _space_list(list_): """ Inserts whitespace between adjacent non-whitespace tokens. """ - spaced_statement = [] # type: List[str] + spaced_statement: List[str] = [] for i in reversed(range(len(list_))): spaced_statement.insert(0, list_[i]) if i > 0 and not list_[i].isspace() and not list_[i-1].isspace(): @@ -205,7 +206,7 @@ class Sentence(Parsable): :returns: whether this lists is parseable by `Sentence`. """ return isinstance(lists, list) and len(lists) > 0 and \ - all(isinstance(elem, str) for elem in lists) + all(isinstance(elem, str) for elem in lists) def parse(self, raw_list, add_spaces=False): """ Parses a list of string types into this object. @@ -213,7 +214,7 @@ class Sentence(Parsable): if add_spaces: raw_list = _space_list(raw_list) if not isinstance(raw_list, list) or \ - any(not isinstance(elem, str) for elem in raw_list): + any(not isinstance(elem, str) for elem in raw_list): raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") self._data = raw_list @@ -270,9 +271,9 @@ class Block(Parsable): contents = [["\n ", "content", " ", "1"], ["\n ", "content", " ", "2"], "\n"] """ def __init__(self, parent=None): - super(Block, self).__init__(parent) - self.names = None # type: Sentence - self.contents = None # type: Block + super().__init__(parent) + self.names: Sentence = None + self.contents: Block = None @staticmethod def should_parse(lists): @@ -284,7 +285,7 @@ class Block(Parsable): :returns: whether this lists is parseable by `Block`. """ return isinstance(lists, list) and len(lists) == 2 and \ - Sentence.should_parse(lists[0]) and isinstance(lists[1], list) + Sentence.should_parse(lists[0]) and isinstance(lists[1], list) def set_tabs(self, tabs=" "): """ Sets tabs by setting equivalent tabbing on names, then adding tabbing diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 385f4cc17..f42a6e85d 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -20,7 +20,7 @@ setup( description="Nginx plugin for Certbot", url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 5e788e394..e6ceca344 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -26,7 +26,7 @@ class NginxConfiguratorTest(util.NginxTest): def setUp(self): - super(NginxConfiguratorTest, self).setUp() + super().setUp() self.config = self.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) @@ -39,7 +39,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(12, len(self.config.parser.parsed)) + self.assertEqual(13, len(self.config.parser.parsed)) @mock.patch("certbot_nginx._internal.configurator.util.exe_exists") @mock.patch("certbot_nginx._internal.configurator.subprocess.Popen") @@ -89,7 +89,7 @@ class NginxConfiguratorTest(util.NginxTest): "155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", - "headers.com", "example.net"}) + "headers.com", "example.net", "ssl.both.com"}) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -935,14 +935,26 @@ class NginxConfiguratorTest(util.NginxTest): prefer_ssl=False, no_ssl_filter_port='80') # Check that the dialog was called with only port 80 vhosts - self.assertEqual(len(mock_select_vhs.call_args[0][0]), 6) + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 8) + + def test_choose_auth_vhosts(self): + """choose_auth_vhosts correctly selects duplicative and HTTP/HTTPS vhosts""" + http, https = self.config.choose_auth_vhosts('ssl.both.com') + self.assertEqual(len(http), 4) + self.assertEqual(len(https), 2) + self.assertEqual(http[0].names, {'ssl.both.com'}) + self.assertEqual(http[1].names, {'ssl.both.com'}) + self.assertEqual(http[2].names, {'ssl.both.com'}) + self.assertEqual(http[3].names, {'*.both.com'}) + self.assertEqual(https[0].names, {'ssl.both.com'}) + self.assertEqual(https[1].names, {'*.both.com'}) class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" def setUp(self): - super(InstallSslOptionsConfTest, self).setUp() + super().setUp() self.config = self.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) diff --git a/certbot-nginx/tests/display_ops_test.py b/certbot-nginx/tests/display_ops_test.py index 377255441..bcd455410 100644 --- a/certbot-nginx/tests/display_ops_test.py +++ b/certbot-nginx/tests/display_ops_test.py @@ -12,7 +12,7 @@ class SelectVhostMultiTest(util.NginxTest): """Tests for certbot_nginx._internal.display_ops.select_vhost_multiple.""" def setUp(self): - super(SelectVhostMultiTest, self).setUp() + super().setUp() nparser = parser.NginxParser(self.config_path) self.vhosts = nparser.get_vhosts() diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index 2947b099d..f10e44859 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -44,10 +44,14 @@ class HttpPerformTest(util.NginxTest): challb=acme_util.chall_to_challb( challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), domain="migration.com", account_key=account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), + domain="ipv6ssl.com", account_key=account_key), ] def setUp(self): - super(HttpPerformTest, self).setUp() + super().setUp() config = self.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) @@ -77,8 +81,8 @@ class HttpPerformTest(util.NginxTest): http_responses = self.http01.perform() - self.assertEqual(len(http_responses), 4) - for i in range(4): + self.assertEqual(len(http_responses), 5) + for i in range(5): self.assertEqual(http_responses[i], acme_responses[i]) def test_mod_config(self): @@ -105,6 +109,43 @@ class HttpPerformTest(util.NginxTest): # self.assertEqual(vhost.addrs, set(v_addr2_print)) # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')])) + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_http_and_https(self, mock_add_server_directives): + """A server_name with both HTTP and HTTPS vhosts should get modded in both vhosts""" + self.configuration.https_port = 443 + self.http01.add_chall(self.achalls[3]) # migration.com + self.http01._mod_config() # pylint: disable=protected-access + + # Domain has an HTTP and HTTPS vhost + # 2 * 'rewrite' + 2 * 'return 200 keyauthz' = 4 + self.assertEqual(mock_add_server_directives.call_count, 4) + + @mock.patch('certbot_nginx._internal.parser.nginxparser.dump') + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_only_https(self, mock_add_server_directives, mock_dump): + """A server_name with only an HTTPS vhost should get modded""" + self.http01.add_chall(self.achalls[4]) # ipv6ssl.com + self.http01._mod_config() # pylint: disable=protected-access + + # It should modify the existing HTTPS vhost + self.assertEqual(mock_add_server_directives.call_count, 2) + # since there was no suitable HTTP vhost or default HTTP vhost, a non-empty one + # should have been created and written to the challenge conf file + self.assertNotEqual(mock_dump.call_args[0][0], []) + + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_deduplicate(self, mock_add_server_directives): + """A vhost that appears in both HTTP and HTTPS vhosts only gets modded once""" + achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), + domain="ssl.both.com", account_key=AUTH_KEY) + self.http01.add_chall(achall) + self.http01._mod_config() # pylint: disable=protected-access + + # Should only get called 5 times, rather than 6, because two vhosts are the same + self.assertEqual(mock_add_server_directives.call_count, 5*2) + @mock.patch("certbot_nginx._internal.configurator.NginxConfigurator.ipv6_info") def test_default_listen_addresses_no_memoization(self, ipv6_info): # pylint: disable=protected-access diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 0083c2448..b062c4196 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -3,6 +3,7 @@ import glob import re import shutil import unittest +from typing import List from certbot import errors from certbot.compat import os @@ -51,6 +52,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual({nparser.abs_path(x) for x in ['foo.conf', 'nginx.conf', 'server.conf', 'sites-enabled/default', + 'sites-enabled/both.com', 'sites-enabled/example.com', 'sites-enabled/headers.com', 'sites-enabled/migration.com', @@ -88,7 +90,7 @@ class NginxParserTest(util.NginxTest): parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(9, len( + self.assertEqual(10, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -111,7 +113,7 @@ class NginxParserTest(util.NginxTest): ([[[0], [3], [4]], [[5], [3], [0]]], [])] for mylist, result in mylists: - paths = [] # type: List[List[int]] + paths: List[List[int]] = [] parser._do_for_subarray(mylist, lambda x: isinstance(x, list) and len(x) >= 1 and @@ -171,7 +173,7 @@ class NginxParserTest(util.NginxTest): '*.www.example.com'}, [], [2, 1, 0]) - self.assertEqual(14, len(vhosts)) + self.assertEqual(19, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] diff --git a/certbot-nginx/tests/test_log_util.py b/certbot-nginx/tests/test_log_util.py deleted file mode 100644 index 7aebf2151..000000000 --- a/certbot-nginx/tests/test_log_util.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Backport for `TestCase.assertLogs()`. - -Most of the idea and code are from CPython implementation. -https://github.com/python/cpython/blob/b76518d43fb82ed9e5d27025d18c90a23d525c90/Lib/unittest/case.py -""" -import logging -import collections - -__all__ = ['AssertLogsMixin'] - -LoggingWatcher = collections.namedtuple('LoggingWatcher', ['records', 'output']) - - -class CapturingHandler(logging.Handler): - """ - A logging handler capturing all (raw and formatted) logging output. - """ - - def __init__(self): - super(CapturingHandler, self).__init__() - self.watcher = LoggingWatcher([], []) - - def flush(self): - pass - - def emit(self, record): - self.watcher.records.append(record) - self.watcher.output.append(self.format(record)) - - - -class AssertLogsContext(object): - """ - A context manager used to implement `TestCase.assertLogs()`. - """ - - LOGGING_FORMAT = '%(levelname)s:%(name)s:%(message)s' - - def __init__(self, test_case, logger_name, level): - self.test_case = test_case - - self.logger_name = logger_name - self.logger_states = None - self.logger = None - - if level: - # pylint: disable=protected-access,no-member - try: - # In Python 3.x - name_to_level = logging._nameToLevel # type: ignore - except AttributeError: - # In Python 2.7 - name_to_level = logging._levelNames # type: ignore - - self.level = name_to_level.get(level, level) - else: - self.level = logging.INFO - - self.watcher = None - - def _save_logger_states(self): - self.logger_states = (self.logger.handlers[:], self.logger.level, self.logger.propagate) - - def _restore_logger_states(self): - self.logger.handlers, self.logger.level, self.logger.propagate = self.logger_states - - def __enter__(self): - if isinstance(self.logger_name, logging.Logger): - self.logger = self.logger_name - else: - self.logger = logging.getLogger(self.logger_name) - - formatter = logging.Formatter(self.LOGGING_FORMAT) - - handler = CapturingHandler() - handler.setFormatter(formatter) - - self._save_logger_states() - self.logger.handlers = [handler] - self.logger.setLevel(self.level) - self.logger.propagate = False - - self.watcher = handler.watcher - return handler.watcher - - def __exit__(self, exc_type, exc_value, tb): - self._restore_logger_states() - - if exc_type is not None: - # let unexpected exceptions pass through - return - - if not self.watcher.records: - self._raiseFailure( - "no logs of level {} or higher triggered on {}" - .format(logging.getLevelName(self.level), self.logger.name)) - - def _raiseFailure(self, message): - message = self.test_case._formatMessage(None, message) # pylint: disable=protected-access - raise self.test_case.failureException(message) - - -class AssertLogsMixin(object): - """ - A mixin that implements `TestCase.assertLogs()`. - """ - - def assertLogs(self, logger=None, level=None): - """Fail unless a log message of level *level* or higher is emitted - on *logger_name* or its children. If omitted, *level* defaults to - INFO and *logger* defaults to the root logger. - This method must be used as a context manager, and will yield - a recording object with two attributes: `output` and `records`. - At the end of the context manager, the `output` attribute will - be a list of the matching formatted log messages and the - `records` attribute will be a list of the corresponding LogRecord - objects. - Example:: - with self.assertLogs('foo', level='INFO') as cm: - logging.getLogger('foo').info('first message') - logging.getLogger('foo.bar').error('second message') - self.assertEqual(cm.output, ['INFO:foo:first message', - 'ERROR:foo.bar:second message']) - """ - return AssertLogsContext(self, logger, level) diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index f545dc5bc..383a15753 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -17,13 +17,12 @@ from certbot.plugins import common from certbot.tests import util as test_util from certbot_nginx._internal import configurator from certbot_nginx._internal import nginxparser -import test_log_util -class NginxTest(test_log_util.AssertLogsMixin, test_util.ConfigTestCase): +class NginxTest(test_util.ConfigTestCase): def setUp(self): - super(NginxTest, self).setUp() + super().setUp() self.configuration = self.config self.config = None diff --git a/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com b/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com new file mode 100644 index 000000000..23a660df3 --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com @@ -0,0 +1,32 @@ +server { + server_name ssl.both.com; +} + +# a duplicate vhost +server { + server_name ssl.both.com; +} + +# a duplicate by means of wildcard +server { + server_name *.both.com; +} + +# combined HTTP and HTTPS +server { + server_name ssl.both.com; + listen 80; + listen 5001 ssl; + + ssl_certificate cert.pem; + ssl_certificate_key cert.key; +} + +# HTTPS, duplicate by means of wildcard +server { + server_name *.both.com; + listen 5001 ssl; + + ssl_certificate cert.pem; + ssl_certificate_key cert.key; +} diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 1067c3b92..fd2f0ffb4 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,59 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.13.0 - master +## 1.16.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + +## 1.15.0 - 2021-05-04 + +### Added + +* + +### Changed + +* + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + +## 1.14.0 - 2021-04-06 + +### Added + +* + +### Changed + +* certbot-auto no longer checks for updates on any operating system. +* The module `acme.magic_typing` is deprecated and will be removed in a future release. + Please use the built-in module `typing` instead. +* The DigitalOcean plugin now creates TXT records for the DNS-01 challenge with a lower 30s TTL. + +### Fixed + +* Don't output an empty line for a hidden certificate when `certbot certificates` is being used + in combination with `--cert-name` or `-d`. + +More details about these changes can be found on our GitHub repo. + +## 1.13.0 - 2021-03-02 ### Added @@ -21,6 +73,10 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). Python 2. * Certbot and all of its components no longer depend on the library `six`. * The update of certbot-auto itself is now disabled on all RHEL-like systems. +* When revoking a certificate by `--cert-name`, it is no longer necessary to specify the `--server` + if the certificate was obtained from a non-default ACME server. +* The nginx authenticator now configures all matching HTTP and HTTPS vhosts for the HTTP-01 + challenge. It is now compatible with external HTTPS redirection by a CDN or load balancer. ### Fixed diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index be06b5803..4557ee399 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.13.0.dev0' +__version__ = '1.16.0.dev0' diff --git a/certbot/certbot/_internal/account.py b/certbot/certbot/_internal/account.py index dbe111fbc..c5667a865 100644 --- a/certbot/certbot/_internal/account.py +++ b/certbot/certbot/_internal/account.py @@ -18,13 +18,13 @@ from certbot import errors from certbot import interfaces from certbot import util from certbot._internal import constants -from certbot.compat import os from certbot.compat import filesystem +from certbot.compat import os logger = logging.getLogger(__name__) -class Account(object): +class Account: """ACME protocol registration. :ivar .RegistrationResource regr: Registration Resource @@ -229,8 +229,7 @@ class AccountFileStorage(interfaces.AccountStorage): def load(self, account_id): return self._load_for_server_path(account_id, self.config.server_path) - def save(self, account, client): - # type: (Account, ClientBase) -> None + def save(self, account: Account, client: ClientBase) -> None: """Create a new account. :param Account account: account to create @@ -245,8 +244,7 @@ class AccountFileStorage(interfaces.AccountStorage): except IOError as error: raise errors.AccountStorageError(error) - def update_regr(self, account, client): - # type: (Account, ClientBase) -> None + def update_regr(self, account: Account, client: ClientBase) -> None: """Update the registration resource. :param Account account: account to update @@ -259,8 +257,7 @@ class AccountFileStorage(interfaces.AccountStorage): except IOError as error: raise errors.AccountStorageError(error) - def update_meta(self, account): - # type: (Account) -> None + def update_meta(self, account: Account) -> None: """Update the meta resource. :param Account account: account to update @@ -338,19 +335,16 @@ class AccountFileStorage(interfaces.AccountStorage): return dir_path - def _prepare(self, account): - # type: (Account) -> str + def _prepare(self, account: Account) -> str: account_dir_path = self._account_dir_path(account.id) util.make_or_verify_dir(account_dir_path, 0o700, self.config.strict_permissions) return account_dir_path - def _create(self, account, dir_path): - # type: (Account, str) -> None + def _create(self, account: Account, dir_path: str) -> None: with util.safe_open(self._key_path(dir_path), "w", chmod=0o400) as key_file: key_file.write(account.key.json_dumps()) - def _update_regr(self, account, acme, dir_path): - # type: (Account, ClientBase, str) -> None + def _update_regr(self, account: Account, acme: ClientBase, dir_path: str) -> None: with open(self._regr_path(dir_path), "w") as regr_file: regr = account.regr # If we have a value for new-authz, save it for forwards diff --git a/certbot/certbot/_internal/auth_handler.py b/certbot/certbot/_internal/auth_handler.py index 7ea2a1de8..c2f323a36 100644 --- a/certbot/certbot/_internal/auth_handler.py +++ b/certbot/certbot/_internal/auth_handler.py @@ -2,15 +2,15 @@ import datetime import logging import time +from typing import Dict +from typing import List +from typing import Tuple import zope.component from acme import challenges from acme import errors as acme_errors from acme import messages -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Tuple from certbot import achallenges from certbot import errors from certbot import interfaces @@ -19,7 +19,7 @@ from certbot._internal import error_handler logger = logging.getLogger(__name__) -class AuthHandler(object): +class AuthHandler: """ACME Authorization Handler for a client. :ivar auth: Authenticator capable of solving @@ -98,8 +98,7 @@ class AuthHandler(object): return authzrs_validated - def deactivate_valid_authorizations(self, orderr): - # type: (messages.OrderResource) -> Tuple[List, List] + def deactivate_valid_authorizations(self, orderr: messages.OrderResource) -> Tuple[List, List]: """ Deactivate all `valid` authorizations in the order, so that they cannot be re-used in subsequent orders. @@ -191,7 +190,7 @@ class AuthHandler(object): """ pending_authzrs = [authzr for authzr in authzrs if authzr.body.status != messages.STATUS_VALID] - achalls = [] # type: List[achallenges.AnnotatedChallenge] + achalls: List[achallenges.AnnotatedChallenge] = [] if pending_authzrs: logger.info("Performing the following challenges:") for authzr in pending_authzrs: @@ -428,7 +427,7 @@ _ERROR_HELP = { def _report_failed_authzrs(failed_authzrs, account_key): """Notifies the user about failed authorizations.""" - problems = {} # type: Dict[str, List[achallenges.AnnotatedChallenge]] + problems: Dict[str, List[achallenges.AnnotatedChallenge]] = {} failed_achalls = [challb_to_achall(challb, account_key, authzr.body.identifier.value) for authzr in failed_authzrs for challb in authzr.body.challenges if challb.error] diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index dfbe4b538..b9b4ad2d7 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -3,11 +3,11 @@ import datetime import logging import re import traceback +from typing import List import pytz import zope.component -from acme.magic_typing import List from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -223,7 +223,7 @@ def cert_path_to_lineage(cli_config): """ acceptable_matches = _acceptable_matches() match = match_and_check_overlaps(cli_config, acceptable_matches, - lambda x: cli_config.cert_path[0], lambda x: x.lineagename) + lambda x: cli_config.cert_path, lambda x: x.lineagename) return match[0] @@ -241,7 +241,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func def find_matches(candidate_lineage, return_value, acceptable_matches): """Returns a list of matches using _search_lineages.""" acceptable_matches = [func(candidate_lineage) for func in acceptable_matches] - acceptable_matches_rv = [] # type: List[str] + acceptable_matches_rv: List[str] = [] for item in acceptable_matches: if isinstance(item, list): acceptable_matches_rv += item @@ -254,7 +254,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func matched = _search_lineages(cli_config, find_matches, [], acceptable_matches) if not matched: - raise errors.Error("No match found for cert-path {0}!".format(cli_config.cert_path[0])) + raise errors.Error("No match found for cert-path {0}!".format(cli_config.cert_path)) elif len(matched) > 1: raise errors.OverlappingMatchFound() return matched @@ -266,9 +266,9 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False): checker = ocsp.RevocationChecker() if config.certname and cert.lineagename != config.certname and not skip_filter_checks: - return "" + return None if config.domains and not set(config.domains).issubset(cert.names()): - return "" + return None now = pytz.UTC.fromutc(datetime.datetime.utcnow()) reasons = [] @@ -358,13 +358,15 @@ def _report_human_readable(config, parsed_certs): """Format a results report for a parsed cert""" certinfo = [] for cert in parsed_certs: - certinfo.append(human_readable_cert_info(config, cert)) + cert_info = human_readable_cert_info(config, cert) + if cert_info is not None: + certinfo.append(cert_info) return "\n".join(certinfo) def _describe_certs(config, parsed_certs, parse_failures): """Print information about the certs we know about""" - out = [] # type: List[str] + out: List[str] = [] notify = out.append diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index 8d2f7c329..7d53ad649 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -1,73 +1,54 @@ """Certbot command line argument & config processing.""" # pylint: disable=too-many-lines -from __future__ import print_function +import argparse import logging import logging.handlers -import argparse import sys -import certbot._internal.plugins.selection as plugin_selection -from certbot._internal.plugins import disco as plugins_disco +from typing import Optional -from acme.magic_typing import Optional - -# pylint: disable=ungrouped-imports import certbot from certbot._internal import constants - -import certbot.plugins.enhancements as enhancements - - -from certbot._internal.cli.cli_constants import ( - LEAUTO, - old_path_fragment, - new_path_prefix, - cli_command, - SHORT_USAGE, - COMMAND_OVERVIEW, - HELP_AND_VERSION_USAGE, - ARGPARSE_PARAMS_TO_REMOVE, - EXIT_ACTIONS, - ZERO_ARG_ACTIONS, - VAR_MODIFIERS, - DEPRECATED_OPTIONS -) - -from certbot._internal.cli.cli_utils import ( - _Default, - read_file, - flag_default, - config_help, - HelpfulArgumentGroup, - CustomHelpFormatter, - _DomainsAction, - add_domains, - CaseInsensitiveList, - _user_agent_comment_type, - _EncodeReasonAction, - parse_preferred_challenges, - _PrefChallAction, - _DeployHookAction, - _RenewHookAction, - nonnegative_int -) - -# These imports depend on cli_constants and cli_utils. -from certbot._internal.cli.verb_help import VERB_HELP, VERB_HELP_MAP +from certbot._internal.cli.cli_constants import ARGPARSE_PARAMS_TO_REMOVE +from certbot._internal.cli.cli_constants import cli_command +from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW +from certbot._internal.cli.cli_constants import DEPRECATED_OPTIONS +from certbot._internal.cli.cli_constants import EXIT_ACTIONS +from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_constants import VAR_MODIFIERS +from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS +from certbot._internal.cli.cli_utils import _Default +from certbot._internal.cli.cli_utils import _DeployHookAction +from certbot._internal.cli.cli_utils import _DomainsAction +from certbot._internal.cli.cli_utils import _EncodeReasonAction +from certbot._internal.cli.cli_utils import _PrefChallAction +from certbot._internal.cli.cli_utils import _RenewHookAction +from certbot._internal.cli.cli_utils import _user_agent_comment_type +from certbot._internal.cli.cli_utils import add_domains +from certbot._internal.cli.cli_utils import CaseInsensitiveList +from certbot._internal.cli.cli_utils import config_help +from certbot._internal.cli.cli_utils import CustomHelpFormatter +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import HelpfulArgumentGroup +from certbot._internal.cli.cli_utils import nonnegative_int +from certbot._internal.cli.cli_utils import parse_preferred_challenges +from certbot._internal.cli.cli_utils import read_file from certbot._internal.cli.group_adder import _add_all_groups -from certbot._internal.cli.subparsers import _create_subparsers +from certbot._internal.cli.helpful import HelpfulArgumentParser from certbot._internal.cli.paths_parser import _paths_parser from certbot._internal.cli.plugins_parsing import _plugins_parsing - -# These imports depend on some or all of the submodules for cli. -from certbot._internal.cli.helpful import HelpfulArgumentParser -# pylint: enable=ungrouped-imports - +from certbot._internal.cli.subparsers import _create_subparsers +from certbot._internal.cli.verb_help import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP_MAP +from certbot._internal.plugins import disco as plugins_disco +import certbot._internal.plugins.selection as plugin_selection +import certbot.plugins.enhancements as enhancements logger = logging.getLogger(__name__) # Global, to save us from a lot of argument passing within the scope of this module -helpful_parser = None # type: Optional[HelpfulArgumentParser] +helpful_parser: Optional[HelpfulArgumentParser] = None def prepare_and_parse_args(plugins, args, detect_defaults=False): @@ -262,8 +243,7 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): " to --server " + constants.STAGING_URI) helpful.add( "testing", "--debug", action="store_true", default=flag_default("debug"), - help="Show tracebacks in case of errors, and allow certbot-auto " - "execution on experimental platforms") + help="Show tracebacks in case of errors") helpful.add( [None, "certonly", "run"], "--debug-challenges", action="store_true", default=flag_default("debug_challenges"), diff --git a/certbot/certbot/_internal/cli/cli_constants.py b/certbot/certbot/_internal/cli/cli_constants.py index bd25f9bee..df815f2e6 100644 --- a/certbot/certbot/_internal/cli/cli_constants.py +++ b/certbot/certbot/_internal/cli/cli_constants.py @@ -1,29 +1,5 @@ """Certbot command line constants""" -import sys - -from certbot.compat import os - -# For help strings, figure out how the user ran us. -# When invoked from letsencrypt-auto, sys.argv[0] is something like: -# "/home/user/.local/share/certbot/bin/certbot" -# Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before -# running letsencrypt-auto (and sudo stops us from seeing if they did), so it -# should only be used for purposes where inability to detect letsencrypt-auto -# fails safely - -LEAUTO = "letsencrypt-auto" -if "CERTBOT_AUTO" in os.environ: - # if we're here, this is probably going to be certbot-auto, unless the - # user saved the script under a different name - LEAUTO = os.path.basename(os.environ["CERTBOT_AUTO"]) - -old_path_fragment = os.path.join(".local", "share", "letsencrypt") -new_path_prefix = os.path.abspath(os.path.join(os.sep, "opt", - "eff.org", "certbot", "venv")) -if old_path_fragment in sys.argv[0] or sys.argv[0].startswith(new_path_prefix): - cli_command = LEAUTO -else: - cli_command = "certbot" +cli_command = "certbot" # Argparse's help formatting has a lot of unhelpful peculiarities, so we want diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index a0ddce38f..8060b5e21 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -5,14 +5,14 @@ import copy import zope.interface.interface # pylint: disable=unused-import from acme import challenges +from certbot import errors from certbot import interfaces from certbot import util -from certbot import errors -from certbot.compat import os from certbot._internal import constants +from certbot.compat import os -class _Default(object): +class _Default: """A class to use as a default to detect if a value is set by a user""" def __bool__(self): @@ -62,11 +62,11 @@ def config_help(name, hidden=False): """Extract the help message for an `.IConfig` attribute.""" if hidden: return argparse.SUPPRESS - field = interfaces.IConfig.__getitem__(name) # type: zope.interface.interface.Attribute + field: zope.interface.interface.Attribute = interfaces.IConfig.__getitem__(name) return field.__doc__ -class HelpfulArgumentGroup(object): +class HelpfulArgumentGroup: """Emulates an argparse group for use with HelpfulArgumentParser. This class is used in the add_group method of HelpfulArgumentParser. @@ -140,7 +140,7 @@ class CaseInsensitiveList(list): through the `helpful` wrapper. It is necessary due to special handling of command line arguments by `set_by_cli` in which the `type_func` is not applied.""" def __contains__(self, element): - return super(CaseInsensitiveList, self).__contains__(element.lower()) + return super().__contains__(element.lower()) def _user_agent_comment_type(value): diff --git a/certbot/certbot/_internal/cli/group_adder.py b/certbot/certbot/_internal/cli/group_adder.py index f22fbc496..0c54c9fe1 100644 --- a/certbot/certbot/_internal/cli/group_adder.py +++ b/certbot/certbot/_internal/cli/group_adder.py @@ -1,6 +1,6 @@ """This module contains a function to add the groups of arguments for the help display""" -from certbot._internal.cli import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP def _add_all_groups(helpful): diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index 80afe5db4..73872584c 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -1,47 +1,42 @@ """Certbot command line argument parser""" -from __future__ import print_function + import argparse import copy import functools import glob import sys +from typing import Any +from typing import Dict import configargparse import zope.component import zope.interface - from zope.interface import interfaces as zope_interfaces -from acme.magic_typing import Any, Dict - from certbot import crypto_util from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import os from certbot._internal import constants from certbot._internal import hooks - +from certbot._internal.cli.cli_constants import ARGPARSE_PARAMS_TO_REMOVE +from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW +from certbot._internal.cli.cli_constants import EXIT_ACTIONS +from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS +from certbot._internal.cli.cli_utils import _Default +from certbot._internal.cli.cli_utils import add_domains +from certbot._internal.cli.cli_utils import CustomHelpFormatter +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import HelpfulArgumentGroup +from certbot._internal.cli.verb_help import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP_MAP +from certbot.compat import os from certbot.display import util as display_util -from certbot._internal.cli import ( - SHORT_USAGE, - CustomHelpFormatter, - flag_default, - VERB_HELP, - VERB_HELP_MAP, - COMMAND_OVERVIEW, - HELP_AND_VERSION_USAGE, - _Default, - add_domains, - EXIT_ACTIONS, - ZERO_ARG_ACTIONS, - ARGPARSE_PARAMS_TO_REMOVE, - HelpfulArgumentGroup -) - -class HelpfulArgumentParser(object): +class HelpfulArgumentParser: """Argparse Wrapper. This class wraps argparse, adding the ability to make --help less @@ -105,9 +100,9 @@ class HelpfulArgumentParser(object): self.visible_topics = self.determine_help_topics(self.help_arg) # elements are added by .add_group() - self.groups = {} # type: Dict[str, argparse._ArgumentGroup] + self.groups: Dict[str, argparse._ArgumentGroup] = {} # elements are added by .parse_args() - self.defaults = {} # type: Dict[str, Any] + self.defaults: Dict[str, Any] = {} self.parser = configargparse.ArgParser( prog="certbot", @@ -121,6 +116,8 @@ class HelpfulArgumentParser(object): # This is the only way to turn off overly verbose config flag documentation self.parser._add_config_file_help = False + self.verb: str + # Help that are synonyms for --help subcommands COMMANDS_TOPICS = ["command", "commands", "subcommand", "subcommands", "verbs"] diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py index 62f5e224d..6197a4bf9 100644 --- a/certbot/certbot/_internal/cli/paths_parser.py +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -1,11 +1,8 @@ """This is a module that adds configuration to the argument parser regarding paths for certificates""" +from certbot._internal.cli.cli_utils import config_help +from certbot._internal.cli.cli_utils import flag_default from certbot.compat import os -from certbot._internal.cli import ( - read_file, - flag_default, - config_help -) def _paths_parser(helpful): @@ -14,22 +11,19 @@ def _paths_parser(helpful): if verb == "help": verb = helpful.help_arg - cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked." - sections = ["paths", "install", "revoke", "certonly", "manage"] + cpkwargs = { + "type": os.path.abspath, + "help": "Path to where certificate is saved (with certonly --csr), installed " + "from, or revoked" + } if verb == "certonly": - add(sections, "--cert-path", type=os.path.abspath, - default=flag_default("auth_cert_path"), help=cph) - elif verb == "revoke": - add(sections, "--cert-path", type=read_file, required=False, help=cph) - else: - add(sections, "--cert-path", type=os.path.abspath, help=cph) + cpkwargs["default"] = flag_default("auth_cert_path") + add(["paths", "install", "revoke", "certonly", "manage"], "--cert-path", **cpkwargs) section = "paths" if verb in ("install", "revoke"): section = verb - # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", - type=((verb == "revoke" and read_file) or os.path.abspath), + add(section, "--key-path", type=os.path.abspath, help="Path to private key for certificate installation " "or revocation (if account key is missing)") diff --git a/certbot/certbot/_internal/cli/plugins_parsing.py b/certbot/certbot/_internal/cli/plugins_parsing.py index 9e11ad3ab..bbfdf22da 100644 --- a/certbot/certbot/_internal/cli/plugins_parsing.py +++ b/certbot/certbot/_internal/cli/plugins_parsing.py @@ -1,5 +1,5 @@ """This is a module that handles parsing of plugins for the argument parser""" -from certbot._internal.cli import flag_default +from certbot._internal.cli.cli_utils import flag_default def _plugins_parsing(helpful, plugins): diff --git a/certbot/certbot/_internal/cli/subparsers.py b/certbot/certbot/_internal/cli/subparsers.py index 13f8705ce..dabb26661 100644 --- a/certbot/certbot/_internal/cli/subparsers.py +++ b/certbot/certbot/_internal/cli/subparsers.py @@ -1,14 +1,11 @@ """This module creates subparsers for the argument parser""" from certbot import interfaces from certbot._internal import constants - -from certbot._internal.cli import ( - flag_default, - read_file, - CaseInsensitiveList, - _user_agent_comment_type, - _EncodeReasonAction -) +from certbot._internal.cli.cli_utils import _EncodeReasonAction +from certbot._internal.cli.cli_utils import _user_agent_comment_type +from certbot._internal.cli.cli_utils import CaseInsensitiveList +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import read_file def _create_subparsers(helpful): @@ -35,8 +32,7 @@ def _create_subparsers(helpful): " Currently --csr only works with the 'certonly' subcommand.") helpful.add("revoke", "--reason", dest="reason", - choices=CaseInsensitiveList(sorted(constants.REVOCATION_REASONS, - key=constants.REVOCATION_REASONS.get)), + choices=CaseInsensitiveList(constants.REVOCATION_REASONS.keys()), action=_EncodeReasonAction, default=flag_default("reason"), help="Specify reason for revoking certificate. (default: unspecified)") helpful.add("revoke", diff --git a/certbot/certbot/_internal/cli/verb_help.py b/certbot/certbot/_internal/cli/verb_help.py index 131cfec96..dfb4e9c67 100644 --- a/certbot/certbot/_internal/cli/verb_help.py +++ b/certbot/certbot/_internal/cli/verb_help.py @@ -1,9 +1,7 @@ """This module contain help information for verbs supported by certbot""" +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_utils import flag_default from certbot.compat import os -from certbot._internal.cli import ( - SHORT_USAGE, - flag_default -) # The attributes here are: # short: a string that will be displayed by "certbot -h commands" diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index 5dc62580e..b5ee73211 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -2,6 +2,7 @@ import datetime import logging import platform +from typing import Optional from cryptography.hazmat.backends import default_backend # See https://github.com/pyca/cryptography/issues/4275 @@ -14,8 +15,6 @@ from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors from acme import messages -from acme.magic_typing import List -from acme.magic_typing import Optional import certbot from certbot import crypto_util from certbot import errors @@ -59,7 +58,7 @@ def determine_user_agent(config): ua = ("CertbotACMEClient/{0} ({1}; {2}{8}) Authenticator/{3} Installer/{4} " "({5}; flags: {6}) Py/{7}") if os.environ.get("CERTBOT_DOCS") == "1": - cli_command = "certbot(-auto)" + cli_command = "certbot" os_info = "OS_NAME OS_VERSION" python_version = "major.minor.patchlevel" else: @@ -93,7 +92,7 @@ def ua_flags(config): flags.append("hook") return " ".join(flags) -class DummyConfig(object): +class DummyConfig: "Shim for computing a sample user agent." def __init__(self): self.authenticator = "XXX" @@ -227,7 +226,7 @@ def perform_registration(acme, config, tos_cb): raise -class Client(object): +class Client: """Certbot's client. :ivar .IConfig config: Client configuration. @@ -255,6 +254,7 @@ class Client(object): acme = acme_from_config_key(config, self.account.key, self.account.regr) self.acme = acme + self.auth_handler: Optional[auth_handler.AuthHandler] if auth is not None: self.auth_handler = auth_handler.AuthHandler( auth, self.acme, self.account, self.config.pref_challs) @@ -328,7 +328,7 @@ class Client(object): with open(old_keypath, "rb") as f: keypath = old_keypath keypem = f.read() - key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key] + key: Optional[util.Key] = util.Key(file=keypath, pem=keypem) logger.info("Reusing existing private key from %s.", old_keypath) else: # The key is set to None here but will be created below. @@ -390,8 +390,8 @@ class Client(object): cert, chain = self.obtain_certificate_from_csr(csr, orderr) return cert, chain, key, csr - def _get_order_and_authorizations(self, csr_pem, best_effort): - # type: (str, bool) -> List[messages.OrderResource] + def _get_order_and_authorizations(self, csr_pem: str, + best_effort: bool) -> messages.OrderResource: """Request a new order and complete its authorizations. :param str csr_pem: A CSR in PEM format. @@ -408,6 +408,9 @@ class Client(object): raise errors.Error("The currently selected ACME CA endpoint does" " not support issuing wildcard certificates.") + if not self.auth_handler: + raise errors.Error("No authorization handler has been set.") + # For a dry run, ensure we have an order with fresh authorizations if orderr and self.config.dry_run: deactivated, failed = self.auth_handler.deactivate_valid_authorizations(orderr) diff --git a/certbot/certbot/_internal/configuration.py b/certbot/certbot/_internal/configuration.py index 1b5cf5da7..aee0022b8 100644 --- a/certbot/certbot/_internal/configuration.py +++ b/certbot/certbot/_internal/configuration.py @@ -13,7 +13,7 @@ from certbot.compat import os @zope.interface.implementer(interfaces.IConfig) -class NamespaceConfig(object): +class NamespaceConfig: """Configuration wrapper around :class:`argparse.Namespace`. For more documentation, including available attributes, please see diff --git a/certbot/certbot/_internal/display/completer.py b/certbot/certbot/_internal/display/completer.py index 03719862b..b43859b19 100644 --- a/certbot/certbot/_internal/display/completer.py +++ b/certbot/certbot/_internal/display/completer.py @@ -1,5 +1,8 @@ """Provides Tab completion when prompting users for a path.""" import glob +from typing import Callable +from typing import Iterator +from typing import Optional # readline module is not available on all systems try: @@ -8,7 +11,7 @@ except ImportError: import certbot._internal.display.dummy_readline as readline # type: ignore -class Completer(object): +class Completer: """Provides Tab completion when prompting users for a path. This class is meant to be used with readline to provide Tab @@ -26,7 +29,9 @@ class Completer(object): """ def __init__(self): - self._iter = self._original_completer = self._original_delims = None + self._iter: Iterator[str] + self._original_completer: Optional[Callable] + self._original_delims: str def complete(self, text, state): """Provides path completion for use with readline. diff --git a/certbot/certbot/_internal/eff.py b/certbot/certbot/_internal/eff.py index 5fbbd302a..b01e2dd61 100644 --- a/certbot/certbot/_internal/eff.py +++ b/certbot/certbot/_internal/eff.py @@ -1,23 +1,21 @@ """Subscribes users to the EFF newsletter.""" import logging +from typing import Optional import requests import zope.component -from acme.magic_typing import Optional # pylint: disable=unused-import - from certbot import interfaces -from certbot.display import util as display_util from certbot._internal import constants -from certbot._internal.account import Account # pylint: disable=unused-import +from certbot._internal.account import Account from certbot._internal.account import AccountFileStorage -from certbot.interfaces import IConfig # pylint: disable=unused-import +from certbot.display import util as display_util +from certbot.interfaces import IConfig logger = logging.getLogger(__name__) -def prepare_subscription(config, acc): - # type: (IConfig, Account) -> None +def prepare_subscription(config: IConfig, acc: Account) -> None: """High level function to store potential EFF newsletter subscriptions. The user may be asked if they want to sign up for the newsletter if @@ -45,8 +43,7 @@ def prepare_subscription(config, acc): storage.update_meta(acc) -def handle_subscription(config, acc): - # type: (IConfig, Account) -> None +def handle_subscription(config: IConfig, acc: Account) -> None: """High level function to take care of EFF newsletter subscriptions. Once subscription is handled, it will not be handled again. @@ -65,8 +62,7 @@ def handle_subscription(config, acc): storage.update_meta(acc) -def _want_subscription(): - # type: () -> bool +def _want_subscription() -> bool: """Does the user want to be subscribed to the EFF newsletter? :returns: True if we should subscribe the user, otherwise, False @@ -83,8 +79,7 @@ def _want_subscription(): return display.yesno(prompt, default=False) -def subscribe(email): - # type: (str) -> None +def subscribe(email: str) -> None: """Subscribe the user to the EFF mailing list. :param str email: the e-mail address to subscribe @@ -99,8 +94,7 @@ def subscribe(email): _check_response(requests.post(url, data=data)) -def _check_response(response): - # type: (requests.Response) -> None +def _check_response(response: requests.Response) -> None: """Check for errors in the server's response. If an error occurred, it will be reported to the user. @@ -120,8 +114,7 @@ def _check_response(response): _report_failure('there was a problem with the server response') -def _report_failure(reason=None): - # type: (Optional[str]) -> None +def _report_failure(reason: Optional[str] = None) -> None: """Notify the user of failing to sign them up for the newsletter. :param reason: a phrase describing what the problem was diff --git a/certbot/certbot/_internal/error_handler.py b/certbot/certbot/_internal/error_handler.py index 60fb287a6..01cc92b42 100644 --- a/certbot/certbot/_internal/error_handler.py +++ b/certbot/certbot/_internal/error_handler.py @@ -3,12 +3,12 @@ import functools import logging import signal import traceback +from typing import Any +from typing import Callable +from typing import Dict +from typing import List +from typing import Union -from acme.magic_typing import Any -from acme.magic_typing import Callable -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Union from certbot import errors from certbot.compat import os @@ -45,7 +45,7 @@ else: _SIGNALS = [] -class ErrorHandler(object): +class ErrorHandler: """Context manager for running code that must be cleaned up on failure. The context manager allows you to register functions that will be called @@ -77,9 +77,9 @@ class ErrorHandler(object): def __init__(self, func, *args, **kwargs): self.call_on_regular_exit = False self.body_executed = False - self.funcs = [] # type: List[Callable[[], Any]] - self.prev_handlers = {} # type: Dict[int, Union[int, None, Callable]] - self.received_signals = [] # type: List[int] + self.funcs: List[Callable[[], Any]] = [] + self.prev_handlers: Dict[int, Union[int, None, Callable]] = {} + self.received_signals: List[int] = [] if func is not None: self.register(func, *args, **kwargs) @@ -108,8 +108,7 @@ class ErrorHandler(object): self._call_signals() return retval - def register(self, func, *args, **kwargs): - # type: (Callable, *Any, **Any) -> None + def register(self, func: Callable, *args: Any, **kwargs: Any) -> None: """Sets func to be run with the given arguments during cleanup. :param function func: function to be called in case of an error diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index 5526b21c4..b9f1f1531 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -1,10 +1,9 @@ """Facilities for implementing hooks that call shell commands.""" -from __future__ import print_function import logging +from typing import List +from typing import Set -from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot import util from certbot.compat import filesystem @@ -78,7 +77,7 @@ def pre_hook(config): _run_pre_hook_if_necessary(cmd) -executed_pre_hooks = set() # type: Set[str] +executed_pre_hooks: Set[str] = set() def _run_pre_hook_if_necessary(command): @@ -128,7 +127,7 @@ def post_hook(config): _run_hook("post-hook", cmd) -post_hooks = [] # type: List[str] +post_hooks: List[str] = [] def _run_eventually(command): diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index aa0b80eaa..80b79d993 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -1,8 +1,8 @@ """Implements file locks compatible with Linux and Windows for locking files and directories.""" import errno import logging +from typing import Optional -from acme.magic_typing import Optional from certbot import errors from certbot.compat import filesystem from certbot.compat import os @@ -16,29 +16,10 @@ else: POSIX_MODE = True - logger = logging.getLogger(__name__) -def lock_dir(dir_path): - # type: (str) -> LockFile - """Place a lock file on the directory at dir_path. - - The lock file is placed in the root of dir_path with the name - .certbot.lock. - - :param str dir_path: path to directory - - :returns: the locked LockFile object - :rtype: LockFile - - :raises errors.LockError: if unable to acquire the lock - - """ - return LockFile(os.path.join(dir_path, '.certbot.lock')) - - -class LockFile(object): +class LockFile: """ Platform independent file lock system. LockFile accepts a parameter, the path to a file acting as a lock. Once the LockFile, @@ -52,8 +33,7 @@ class LockFile(object): LockFile is platform independent: it will proceed to the appropriate OS lock mechanism depending on Linux or Windows. """ - def __init__(self, path): - # type: (str) -> None + def __init__(self, path: str) -> None: """ Create a LockFile instance on the given file path, and acquire lock. :param str path: the path to the file that will hold a lock @@ -64,8 +44,7 @@ class LockFile(object): self.acquire() - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) if self.is_locked(): repr_str += 'acquired>' @@ -73,23 +52,20 @@ class LockFile(object): repr_str += 'released>' return repr_str - def acquire(self): - # type: () -> None + def acquire(self) -> None: """ Acquire the lock on the file, forbidding any other Certbot instance to acquire it. :raises errors.LockError: if unable to acquire the lock """ self._lock_mechanism.acquire() - def release(self): - # type: () -> None + def release(self) -> None: """ Release the lock on the file, allowing any other Certbot instance to acquire it. """ self._lock_mechanism.release() - def is_locked(self): - # type: () -> bool + def is_locked(self) -> bool: """ Check if the file is currently locked. :return: True if the file is locked, False otherwise @@ -97,18 +73,16 @@ class LockFile(object): return self._lock_mechanism.is_locked() -class _BaseLockMechanism(object): - def __init__(self, path): - # type: (str) -> None +class _BaseLockMechanism: + def __init__(self, path: str) -> None: """ Create a lock file mechanism for Unix. :param str path: the path to the lock file """ self._path = path - self._fd = None # type: Optional[int] + self._fd: Optional[int] = None - def is_locked(self): - # type: () -> bool + def is_locked(self) -> bool: """Check if lock file is currently locked. :return: True if the lock file is locked :rtype: bool @@ -129,8 +103,7 @@ class _UnixLockMechanism(_BaseLockMechanism): process exits. It cannot be used to provide synchronization between threads. It is based on the lock_file package by Martin Horcicka. """ - def acquire(self): - # type: () -> None + def acquire(self) -> None: """Acquire the lock.""" while self._fd is None: # Open the file @@ -144,8 +117,7 @@ class _UnixLockMechanism(_BaseLockMechanism): if self._fd is None: os.close(fd) - def _try_lock(self, fd): - # type: (int) -> None + def _try_lock(self, fd: int) -> None: """ Try to acquire the lock file without blocking. :param int fd: file descriptor of the opened file to lock @@ -158,8 +130,7 @@ class _UnixLockMechanism(_BaseLockMechanism): raise errors.LockError('Another instance of Certbot is already running.') raise - def _lock_success(self, fd): - # type: (int) -> bool + def _lock_success(self, fd: int) -> bool: """ Did we successfully grab the lock? Because this class deletes the locked file when the lock is @@ -185,8 +156,7 @@ class _UnixLockMechanism(_BaseLockMechanism): # the same device and inode, they're the same file. return stat1.st_dev == stat2.st_dev and stat1.st_ino == stat2.st_ino - def release(self): - # type: () -> None + def release(self) -> None: """Remove, close, and release the lock file.""" # It is important the lock file is removed before it's released, # otherwise: @@ -235,10 +205,9 @@ class _WindowsLockMechanism(_BaseLockMechanism): # Under Windows, filesystem.open will raise directly an EACCES error # if the lock file is already locked. fd = filesystem.open(self._path, open_mode, 0o600) - # The need for this "type: ignore" was fixed in - # https://github.com/python/typeshed/pull/3607 and included in - # newer versions of mypy so it can be removed when mypy is - # upgraded. + # This "type: ignore" is currently needed because msvcrt methods + # are only defined on Windows. See + # https://github.com/python/typeshed/blob/16ae4c61201cd8b96b8b22cdfb2ab9e89ba5bcf2/stdlib/msvcrt.pyi. msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) # type: ignore except (IOError, OSError) as err: if fd: @@ -254,10 +223,11 @@ class _WindowsLockMechanism(_BaseLockMechanism): def release(self): """Release the lock.""" try: - # The need for this "type: ignore" was fixed in - # https://github.com/python/typeshed/pull/3607 and included in - # newer versions of mypy so it can be removed when mypy is - # upgraded. + if not self._fd: + raise errors.Error("The lock has not been acquired first.") + # This "type: ignore" is currently needed because msvcrt methods + # are only defined on Windows. See + # https://github.com/python/typeshed/blob/16ae4c61201cd8b96b8b22cdfb2ab9e89ba5bcf2/stdlib/msvcrt.pyi. msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) # type: ignore os.close(self._fd) @@ -269,3 +239,20 @@ class _WindowsLockMechanism(_BaseLockMechanism): logger.debug(str(e)) finally: self._fd = None + + +def lock_dir(dir_path: str) -> LockFile: + """Place a lock file on the directory at dir_path. + + The lock file is placed in the root of dir_path with the name + .certbot.lock. + + :param str dir_path: path to directory + + :returns: the locked LockFile object + :rtype: LockFile + + :raises errors.LockError: if unable to acquire the lock + + """ + return LockFile(os.path.join(dir_path, '.certbot.lock')) diff --git a/certbot/certbot/_internal/log.py b/certbot/certbot/_internal/log.py index 04a8cb0d1..700f9e9f4 100644 --- a/certbot/certbot/_internal/log.py +++ b/certbot/certbot/_internal/log.py @@ -19,7 +19,7 @@ The preferred method to display important information to the user is to use `certbot.display.util` and `certbot.display.ops`. """ -from __future__ import print_function + import functools import logging @@ -112,11 +112,12 @@ def post_arg_parse_setup(config): root_logger.addHandler(file_handler) root_logger.removeHandler(memory_handler) - temp_handler = memory_handler.target # pylint: disable=no-member + temp_handler = getattr(memory_handler, 'target', None) memory_handler.setTarget(file_handler) # pylint: disable=no-member memory_handler.flush(force=True) # pylint: disable=unexpected-keyword-arg memory_handler.close() - temp_handler.close() + if temp_handler: + temp_handler.close() if config.quiet: level = constants.QUIET_LOGGING_LEVEL @@ -176,7 +177,7 @@ class ColoredStreamHandler(logging.StreamHandler): """ def __init__(self, stream=None): - super(ColoredStreamHandler, self).__init__(stream) + super().__init__(stream) self.colored = (sys.stderr.isatty() if stream is None else stream.isatty()) self.red_level = logging.WARNING @@ -190,7 +191,7 @@ class ColoredStreamHandler(logging.StreamHandler): :rtype: str """ - out = super(ColoredStreamHandler, self).format(record) + out = super().format(record) if self.colored and record.levelno >= self.red_level: return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) return out @@ -205,14 +206,14 @@ class MemoryHandler(logging.handlers.MemoryHandler): """ def __init__(self, target=None, capacity=10000): # capacity doesn't matter because should_flush() is overridden - super(MemoryHandler, self).__init__(capacity, target=target) + super().__init__(capacity, target=target) def close(self): """Close the memory handler, but don't set the target to None.""" # This allows the logging module which may only have a weak # reference to the target handler to properly flush and close it. - target = self.target - super(MemoryHandler, self).close() + target = getattr(self, 'target') + super().close() self.target = target def flush(self, force=False): # pylint: disable=arguments-differ @@ -226,7 +227,7 @@ class MemoryHandler(logging.handlers.MemoryHandler): # This method allows flush() calls in logging.shutdown to be a # noop so we can control when this handler is flushed. if force: - super(MemoryHandler, self).flush() + super().flush() def shouldFlush(self, record): """Should the buffer be automatically flushed? @@ -254,7 +255,7 @@ class TempHandler(logging.StreamHandler): self._workdir = tempfile.mkdtemp() self.path = os.path.join(self._workdir, 'log') stream = util.safe_open(self.path, mode='w', chmod=0o600) - super(TempHandler, self).__init__(stream) + super().__init__(stream) self._delete = True def emit(self, record): @@ -264,7 +265,7 @@ class TempHandler(logging.StreamHandler): """ self._delete = False - super(TempHandler, self).emit(record) + super().emit(record) def close(self): """Close the handler and the temporary log file. @@ -280,7 +281,7 @@ class TempHandler(logging.StreamHandler): if self._delete: shutil.rmtree(self._workdir) self._delete = False - super(TempHandler, self).close() + super().close() finally: self.release() diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index b9b6b16f6..aed265ba3 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -1,17 +1,23 @@ """Certbot main entry point.""" # pylint: disable=too-many-lines -from __future__ import print_function import functools import logging.handlers import sys +from contextlib import contextmanager +from typing import Generator +from typing import IO +from typing import Iterable +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union import configobj import josepy as jose import zope.component from acme import errors as acme_errors -from acme.magic_typing import Union, Iterable, Optional, List, Tuple # pylint: disable=unused-import import certbot from certbot import crypto_util from certbot import errors @@ -142,8 +148,8 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N return lineage -def _handle_unexpected_key_type_migration(config, cert): - # type: (configuration.NamespaceConfig, storage.RenewableCert) -> None +def _handle_unexpected_key_type_migration(config: configuration.NamespaceConfig, + cert: storage.RenewableCert) -> None: """ This function ensures that the user will not implicitly migrate an existing key from one type to another in the situation where a certificate for that lineage @@ -164,11 +170,10 @@ def _handle_unexpected_key_type_migration(config, cert): raise errors.Error(msg) -def _handle_subset_cert_request(config, # type: configuration.NamespaceConfig - domains, # type: List[str] - cert # type: storage.RenewableCert - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _handle_subset_cert_request(config: configuration.NamespaceConfig, + domains: List[str], + cert: storage.RenewableCert + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Figure out what to do if a previous cert had a subset of the names now requested :param config: Configuration object @@ -215,10 +220,9 @@ def _handle_subset_cert_request(config, # type: configuration.NamespaceConfig raise errors.Error(USER_CANCELLED) -def _handle_identical_cert_request(config, # type: configuration.NamespaceConfig - lineage, # type: storage.RenewableCert - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _handle_identical_cert_request(config: configuration.NamespaceConfig, + lineage: storage.RenewableCert, + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Figure out what to do if a lineage has the same names as a previously obtained one :param config: Configuration object @@ -334,11 +338,10 @@ def _find_cert(config, domains, certname): return (action != "reinstall"), lineage -def _find_lineage_for_domains_and_certname(config, # type: configuration.NamespaceConfig - domains, # type: List[str] - certname # type: str - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _find_lineage_for_domains_and_certname(config: configuration.NamespaceConfig, + domains: List[str], + certname: str + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Find appropriate lineage based on given domains and/or certname. :param config: Configuration object @@ -755,7 +758,7 @@ def update_account(config, unused_plugins): cb_client = client.Client(config, acc, None, None, acme=acme) # Empty list of contacts in case the user is removing all emails - acc_contacts = () # type: Iterable[str] + acc_contacts: Iterable[str] = () if config.email: acc_contacts = ['mailto:' + email for email in config.email.split(',')] # We rely on an exception to interrupt this process if it didn't work. @@ -1072,8 +1075,7 @@ def certificates(config, unused_plugins): cert_manager.certificates(config) -# TODO: coop with renewal config -def revoke(config, unused_plugins): +def revoke(config, unused_plugins: plugins_disco.PluginsRegistry) -> Optional[str]: """Revoke a previously obtained certificate. :param config: Configuration object @@ -1090,7 +1092,13 @@ def revoke(config, unused_plugins): config.installer = config.authenticator = None if config.cert_path is None and config.certname: - config.cert_path = storage.cert_path_for_cert_name(config, config.certname) + # When revoking via --cert-name, take the cert path and server from renewalparams + lineage = storage.RenewableCert( + storage.renewal_file_for_certname(config, config.certname), config) + config.cert_path = lineage.cert_path + # --server takes priority over lineage.server + if lineage.server and not cli.set_by_cli("server"): + config.server = lineage.server elif not config.cert_path or (config.cert_path and config.certname): # intentionally not supporting --cert-path & --cert-name together, # to avoid dealing with mismatched values @@ -1098,15 +1106,18 @@ def revoke(config, unused_plugins): if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using certificate key %s", - config.cert_path[0], config.key_path[0]) - crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) - key = jose.JWK.load(config.key_path[1]) + config.cert_path, config.key_path) + crypto_util.verify_cert_matches_priv_key(config.cert_path, config.key_path) + with open(config.key_path, 'rb') as f: + key = jose.JWK.load(f.read()) acme = client.acme_from_config_key(config, key) else: # revocation by account key - logger.debug("Revoking %s using Account Key", config.cert_path[0]) + logger.debug("Revoking %s using Account Key", config.cert_path) acc, _ = _determine_account(config) acme = client.acme_from_config_key(config, acc.key, acc.regr) - cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] + + with open(config.cert_path, 'rb') as f: + cert = crypto_util.pyopenssl_load_certificate(f.read())[0] logger.debug("Reason code for revocation: %s", config.reason) try: acme.revoke(jose.ComparableX509(cert), config.reason) @@ -1114,7 +1125,7 @@ def revoke(config, unused_plugins): except acme_errors.ClientError as e: return str(e) - display_ops.success_revocation(config.cert_path[0]) + display_ops.success_revocation(config.cert_path) return None @@ -1339,26 +1350,36 @@ def make_or_verify_needed_dirs(config): util.make_or_verify_dir(hook_dir, strict=config.strict_permissions) -def set_displayer(config): - """Set the displayer +@contextmanager +def make_displayer(config: configuration.NamespaceConfig + ) -> Generator[Union[display_util.NoninteractiveDisplay, + display_util.FileDisplay], None, None]: + """Creates a display object appropriate to the flags in the supplied config. :param config: Configuration object - :type config: interfaces.IConfig - :returns: `None` - :rtype: None + :returns: Display object implementing :class:`certbot.interfaces.IDisplay` """ + displayer: Union[None, display_util.NoninteractiveDisplay, + display_util.FileDisplay] = None + devnull: Optional[IO] = None + if config.quiet: config.noninteractive_mode = True - displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) \ - # type: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] + devnull = open(os.devnull, "w") + displayer = display_util.NoninteractiveDisplay(devnull) elif config.noninteractive_mode: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: - displayer = display_util.FileDisplay(sys.stdout, - config.force_interactive) - zope.component.provideUtility(displayer) + displayer = display_util.FileDisplay( + sys.stdout, config.force_interactive) + + try: + yield displayer + finally: + if devnull: + devnull.close() def main(cli_args=None): @@ -1403,11 +1424,12 @@ def main(cli_args=None): if config.func != plugins_cmd: # pylint: disable=comparison-with-callable raise - set_displayer(config) - # Reporter report = reporter.Reporter(config) zope.component.provideUtility(report) util.atexit_register(report.print_messages) - return config.func(config, plugins) + with make_displayer(config) as displayer: + zope.component.provideUtility(displayer) + + return config.func(config, plugins) diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index dbcecb067..744062968 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -1,24 +1,20 @@ """Utilities for plugins discovery and selection.""" -import collections import itertools import logging import sys +from collections.abc import Mapping +from typing import Dict +from typing import Optional +from typing import Union import pkg_resources import zope.interface import zope.interface.verify - -from acme.magic_typing import Dict from certbot import errors from certbot import interfaces from certbot._internal import constants from certbot.compat import os - -try: - # Python 3.3+ - from collections.abc import Mapping -except ImportError: # pragma: no cover - from collections import Mapping +from certbot.errors import Error logger = logging.getLogger(__name__) @@ -44,21 +40,21 @@ PREFIX_FREE_DISTRIBUTIONS = [ """Distributions for which prefix will be omitted.""" -class PluginEntryPoint(object): +class PluginEntryPoint: """Plugin entry point.""" # this object is mutable, don't allow it to be hashed! __hash__ = None # type: ignore - def __init__(self, entry_point, with_prefix=False): + def __init__(self, entry_point: pkg_resources.EntryPoint, with_prefix=False): self.name = self.entry_point_to_plugin_name(entry_point, with_prefix) - self.plugin_cls = entry_point.load() + self.plugin_cls: interfaces.IPluginFactory = entry_point.load() self.entry_point = entry_point - self.warning_message = None - self._initialized = None - self._prepared = None + self.warning_message: Optional[str] = None + self._initialized: Optional[interfaces.IPlugin] = None + self._prepared: Optional[Union[bool, Error]] = None self._hidden = False - self._long_description = None + self._long_description: Optional[str] = None def check_name(self, name): """Check if the name refers to this plugin.""" @@ -124,12 +120,15 @@ class PluginEntryPoint(object): """Memoized plugin initialization.""" if not self.initialized: self.entry_point.require() # fetch extras! - self._initialized = self.plugin_cls(config, self.name) + # TODO: remove type ignore once the interface becomes a proper + # abstract class (using abc) that mypy understands. + self._initialized = self.plugin_cls(config, self.name) # type: ignore return self._initialized def verify(self, ifaces): """Verify that the plugin conforms to the specified interfaces.""" - assert self.initialized + if not self.initialized: + raise ValueError("Plugin is not initialized.") for iface in ifaces: # zope.interface.providedBy(plugin) try: zope.interface.verify.verifyObject(iface, self.init()) @@ -150,10 +149,13 @@ class PluginEntryPoint(object): def prepare(self): """Memoized plugin preparation.""" - assert self.initialized + if self._initialized is None: + raise ValueError("Plugin is not initialized.") if self._prepared is None: try: - self._initialized.prepare() + # TODO: remove type ignore once the interface becomes a proper + # abstract class (using abc) that mypy understands. + self._initialized.prepare() # type: ignore except errors.MisconfigurationError as error: logger.debug("Misconfigured %r: %s", self, error, exc_info=True) self._prepared = error @@ -214,12 +216,12 @@ class PluginsRegistry(Mapping): # This prevents deadlock caused by plugins acquiring a lock # and ensures at least one concurrent Certbot instance will run # successfully. - self._plugins = collections.OrderedDict(sorted(plugins.items())) + self._plugins = dict(sorted(plugins.items())) @classmethod def find_all(cls): """Find plugins using setuptools entry points.""" - plugins = {} # type: Dict[str, PluginEntryPoint] + plugins: Dict[str, PluginEntryPoint] = {} plugin_paths_string = os.getenv('CERTBOT_PLUGIN_PATH') plugin_paths = plugin_paths_string.split(':') if plugin_paths_string else [] # XXX should ensure this only happens once @@ -253,8 +255,10 @@ class PluginsRegistry(Mapping): plugin_ep = PluginEntryPoint(entry_point, with_prefix) if plugin_ep.name in plugins: other_ep = plugins[plugin_ep.name] + plugin1 = plugin_ep.entry_point.dist.key if plugin_ep.entry_point.dist else "unknown" + plugin2 = other_ep.entry_point.dist.key if other_ep.entry_point.dist else "unknown" raise Exception("Duplicate plugin name {0} from {1} and {2}.".format( - plugin_ep.name, plugin_ep.entry_point.dist.key, other_ep.entry_point.dist.key)) + plugin_ep.name, plugin1, plugin2)) if interfaces.IPluginFactory.providedBy(plugin_ep.plugin_cls): plugins[plugin_ep.name] = plugin_ep else: # pragma: no cover diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index a2e4bb28e..dec73a1ed 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -1,10 +1,11 @@ """Manual authenticator plugin""" +from typing import Dict + import zope.component import zope.interface from acme import challenges -from acme.magic_typing import Dict -from certbot import achallenges # pylint: disable=unused-import +from certbot import achallenges from certbot import errors from certbot import interfaces from certbot import reverter @@ -42,13 +43,30 @@ class Authenticator(common.Plugin): '$CERTBOT_REMAINING_CHALLENGES will be equal to the number of challenges that ' 'remain after the current one, and $CERTBOT_ALL_DOMAINS contains a comma-separated ' 'list of all domains that are challenged for the current certificate.') + # Include the full stop at the end of the FQDN in the instructions below for the null + # label of the DNS root, as stated in section 3.1 of RFC 1035. While not necessary + # for most day to day usage of hostnames, when adding FQDNs to a DNS zone editor, this + # full stop is often mandatory. Without a full stop, the entered name is often seen as + # relative to the DNS zone origin, which could lead to entries for, e.g.: + # _acme-challenge.example.com.example.com. For users unaware of this subtle detail, + # including the trailing full stop in the DNS instructions below might avert this issue. _DNS_INSTRUCTIONS = """\ -Please deploy a DNS TXT record under the name -{domain} with the following value: +Please deploy a DNS TXT record under the name: + +{domain}. + +with the following value: {validation} - -Before continuing, verify the record is deployed.""" +""" + _DNS_VERIFY_INSTRUCTIONS = """ +Before continuing, verify the TXT record has been deployed. Depending on the DNS +provider, this may take some time, from a few seconds to multiple minutes. You can +check if it has finished deploying with aid of online tools, such as the Google +Admin Toolbox: https://toolbox.googleapps.com/apps/dig/#TXT/{domain}. +Look for one or more bolded line(s) below the line ';ANSWER'. It should show the +value(s) you've just added. +""" _HTTP_INSTRUCTIONS = """\ Create a file containing just this data: @@ -70,11 +88,10 @@ permitted by DNS standards.) """ def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() - self.env = {} \ - # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] + self.env: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] = {} self.subsequent_dns_challenge = False self.subsequent_any_challenge = False @@ -114,11 +131,15 @@ permitted by DNS standards.) def perform(self, achalls): # pylint: disable=missing-function-docstring responses = [] - for achall in achalls: + last_dns_achall = 0 + for i, achall in enumerate(achalls): + if isinstance(achall.chall, challenges.DNS01): + last_dns_achall = i + for i, achall in enumerate(achalls): if self.conf('auth-hook'): self._perform_achall_with_script(achall, achalls) else: - self._perform_achall_manually(achall) + self._perform_achall_manually(achall, i == last_dns_achall) responses.append(achall.response(achall.account_key)) return responses @@ -136,7 +157,7 @@ permitted by DNS standards.) env['CERTBOT_AUTH_OUTPUT'] = out.strip() self.env[achall] = env - def _perform_achall_manually(self, achall): + def _perform_achall_manually(self, achall, last_dns_achall=False): validation = achall.validation(achall.account_key) if isinstance(achall.chall, challenges.HTTP01): msg = self._HTTP_INSTRUCTIONS.format( @@ -152,7 +173,15 @@ permitted by DNS standards.) if self.subsequent_dns_challenge: # 2nd or later dns-01 challenge msg += self._SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS + elif self.subsequent_any_challenge: + # 1st dns-01 challenge, but 2nd or later *any* challenge, so + # instruct user not to remove any previous http-01 challenge + msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS self.subsequent_dns_challenge = True + if last_dns_achall: + # last dns-01 challenge + msg += self._DNS_VERIFY_INSTRUCTIONS.format( + domain=achall.validation_domain_name(achall.domain)) elif self.subsequent_any_challenge: # 2nd or later challenge of another type msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index e5c311efe..f0cce002f 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -1,5 +1,4 @@ """Decide which plugins to use for authentication & installation""" -from __future__ import print_function import logging @@ -135,19 +134,10 @@ def choose_plugin(prepared, question): opts = [plugin_ep.description_with_name + (" [Misconfigured]" if plugin_ep.misconfigured else "") for plugin_ep in prepared] - names = set(plugin_ep.name for plugin_ep in prepared) while True: disp = z_util(interfaces.IDisplay) - if "CERTBOT_AUTO" in os.environ and names == {"apache", "nginx"}: - # The possibility of being offered exactly apache and nginx here - # is new interactivity brought by https://github.com/certbot/certbot/issues/4079, - # so set apache as a default for those kinds of non-interactive use - # (the user will get a warning to set --non-interactive or --force-interactive) - apache_idx = [n for n, p in enumerate(prepared) if p.name == "apache"][0] - code, index = disp.menu(question, opts, default=apache_idx) - else: - code, index = disp.menu(question, opts, force_interactive=True) + code, index = disp.menu(question, opts, force_interactive=True) if code == display_util.OK: plugin_ep = prepared[index] diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index 60c9558cb..5fb29671f 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -1,20 +1,19 @@ """Standalone Authenticator.""" import collections +import errno import logging import socket -# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi -from socket import errno as socket_errors # type: ignore +from typing import DefaultDict +from typing import Dict +from typing import Set +from typing import Tuple +from typing import TYPE_CHECKING -import OpenSSL # pylint: disable=unused-import +import OpenSSL import zope.interface from acme import challenges from acme import standalone as acme_standalone -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import Set -from acme.magic_typing import Tuple -from acme.magic_typing import TYPE_CHECKING from certbot import achallenges from certbot import errors from certbot import interfaces @@ -28,7 +27,7 @@ if TYPE_CHECKING: Set[achallenges.KeyAuthorizationAnnotatedChallenge] ] -class ServerManager(object): +class ServerManager: """Standalone servers manager. Manager for `ACMEServer` and `ACMETLSServer` instances. @@ -42,7 +41,7 @@ class ServerManager(object): """ def __init__(self, certs, http_01_resources): - self._instances = {} # type: Dict[int, acme_standalone.BaseDualNetworkedServers] + self._instances: Dict[int, acme_standalone.BaseDualNetworkedServers] = {} self.certs = certs self.http_01_resources = http_01_resources @@ -120,17 +119,16 @@ class Authenticator(common.Plugin): description = "Spin up a temporary webserver" def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) - self.served = collections.defaultdict(set) # type: ServedType + self.served: ServedType = collections.defaultdict(set) # Stuff below is shared across threads (i.e. servers read # values, main thread writes). Due to the nature of CPython's # GIL, the operations are safe, c.f. # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe - self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] - self.http_01_resources = set() \ - # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] + self.certs: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] = {} + self.http_01_resources: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] = set() self.servers = ServerManager(self.certs, self.http_01_resources) @@ -188,13 +186,13 @@ class Authenticator(common.Plugin): def _handle_perform_error(error): - if error.socket_error.errno == socket_errors.EACCES: + if error.socket_error.errno == errno.EACCES: raise errors.PluginError( "Could not bind TCP port {0} because you don't have " "the appropriate permissions (for example, you " "aren't running this program as " "root).".format(error.port)) - if error.socket_error.errno == socket_errors.EADDRINUSE: + if error.socket_error.errno == errno.EADDRINUSE: display = zope.component.getUtility(interfaces.IDisplay) msg = ( "Could not bind TCP port {0} because it is already in " diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index 8789db604..3473d3c8e 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -3,19 +3,19 @@ import argparse import collections import json import logging +from typing import DefaultDict +from typing import Dict +from typing import List +from typing import Set import zope.component import zope.interface from acme import challenges -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces from certbot._internal import cli +from certbot.achallenges import KeyAuthorizationAnnotatedChallenge as AnnotatedChallenge from certbot.compat import filesystem from certbot.compat import os from certbot.display import ops @@ -66,12 +66,11 @@ to serve all files under specified web root ({0}).""" return [challenges.HTTP01] def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.full_roots = {} # type: Dict[str, str] - self.performed = collections.defaultdict(set) \ - # type: DefaultDict[str, Set[achallenges.KeyAuthorizationAnnotatedChallenge]] + super().__init__(*args, **kwargs) + self.full_roots: Dict[str, str] = {} + self.performed: DefaultDict[str, Set[AnnotatedChallenge]] = collections.defaultdict(set) # stack of dirs successfully created by this authenticator - self._created_dirs = [] # type: List[str] + self._created_dirs: List[str] = [] def prepare(self): # pylint: disable=missing-function-docstring pass @@ -224,7 +223,7 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - not_removed = [] # type: List[str] + not_removed: List[str] = [] while self._created_dirs: path = self._created_dirs.pop() try: @@ -251,7 +250,7 @@ class _WebrootPathAction(argparse.Action): """Action class for parsing webroot_path.""" def __init__(self, *args, **kwargs): - super(_WebrootPathAction, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._domain_before_webroot = False def __call__(self, parser, namespace, webroot_path, option_string=None): diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index 9fe9cb546..f29709ef4 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -1,5 +1,4 @@ """Functionality for autorenewal and associated juggling of configurations""" -from __future__ import print_function import copy import itertools @@ -8,28 +7,29 @@ import random import sys import time import traceback +from typing import List +from typing import Optional from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import ec, rsa +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.serialization import load_pem_private_key import OpenSSL import zope.component -from acme.magic_typing import List -from acme.magic_typing import Optional # pylint: disable=unused-import from certbot import crypto_util -from certbot.display import util as display_util from certbot import errors from certbot import interfaces from certbot import util from certbot._internal import cli -from certbot._internal import client # pylint: disable=unused-import +from certbot._internal import client from certbot._internal import constants from certbot._internal import hooks from certbot._internal import storage from certbot._internal import updater from certbot._internal.plugins import disco as plugins_disco from certbot.compat import os +from certbot.display import util as display_util logger = logging.getLogger(__name__) @@ -143,7 +143,7 @@ def _restore_plugin_configs(config, renewalparams): # longer defined, stored copies of that parameter will be # deserialized as strings by this logic even if they were # originally meant to be some other type. - plugin_prefixes = [] # type: List[str] + plugin_prefixes: List[str] = [] if renewalparams["authenticator"] == "webroot": _restore_webroot_config(config, renewalparams) else: @@ -290,7 +290,7 @@ def _restore_str(name, value): def should_renew(config, lineage): - "Return true if any of the circumstances for automatic renewal apply." + """Return true if any of the circumstances for automatic renewal apply.""" if config.renew_by_default: logger.debug("Auto-renewal forced with --force-renewal...") return True @@ -305,19 +305,16 @@ def should_renew(config, lineage): def _avoid_invalidating_lineage(config, lineage, original_server): - "Do not renew a valid cert with one from a staging server!" + """Do not renew a valid cert with one from a staging server!""" # Some lineages may have begun with --staging, but then had production # certificates added to them with open(lineage.cert) as the_file: contents = the_file.read() latest_cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, contents) - # all our test certificates are from happy hacker fake CA, though maybe one day - # we should test more methodically - now_valid = "fake" not in repr(latest_cert.get_issuer()).lower() if util.is_staging(config.server): - if not util.is_staging(original_server) or now_valid: + if not util.is_staging(original_server): if not config.break_my_certs: names = ", ".join(lineage.names()) raise errors.Error( @@ -326,8 +323,8 @@ def _avoid_invalidating_lineage(config, lineage, original_server): "unless you use the --break-my-certs flag!".format(names)) -def renew_cert(config, domains, le_client, lineage): - # type: (interfaces.IConfig, Optional[List[str]], client.Client, storage.RenewableCert) -> None +def renew_cert(config: interfaces.IConfig, domains: Optional[List[str]], le_client: client.Client, + lineage: storage.RenewableCert) -> None: """Renew a certificate lineage.""" renewal_params = lineage.configuration["renewalparams"] original_server = renewal_params.get("server", cli.flag_default("server")) @@ -354,14 +351,14 @@ def renew_cert(config, domains, le_client, lineage): def report(msgs, category): - "Format a results report for a category of renewal outcomes" + """Format a results report for a category of renewal outcomes""" lines = ("%s (%s)" % (m, category) for m in msgs) return " " + "\n ".join(lines) -def _renew_describe_results(config, renew_successes, renew_failures, - renew_skipped, parse_failures): - # type: (interfaces.IConfig, List[str], List[str], List[str], List[str]) -> None +def _renew_describe_results(config: interfaces.IConfig, renew_successes: List[str], + renew_failures: List[str], renew_skipped: List[str], + parse_failures: List[str]) -> None: """ Print a report to the terminal about the results of the renewal process. @@ -514,8 +511,7 @@ def handle_renewal_request(config): logger.debug("no renewal failures") -def _update_renewal_params_from_key(key_path, config): - # type: (str, interfaces.IConfig) -> None +def _update_renewal_params_from_key(key_path: str, config: interfaces.IConfig) -> None: with open(key_path, 'rb') as file_h: key = load_pem_private_key(file_h.read(), password=None, backend=default_backend()) if isinstance(key, rsa.RSAPrivateKey): diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index 64c0fbd6d..295a2d4c5 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -1,6 +1,4 @@ """Collects and displays information to the user.""" -from __future__ import print_function - import collections import logging import queue @@ -16,7 +14,7 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IReporter) -class Reporter(object): +class Reporter: """Collects and displays information to the user. :ivar `queue.PriorityQueue` messages: Messages to be displayed to @@ -31,10 +29,10 @@ class Reporter(object): LOW_PRIORITY = 2 """Low priority constant. See `add_message`.""" - _msg_type = collections.namedtuple('ReporterMsg', 'priority text on_crash') + _msg_type = collections.namedtuple('_msg_type', 'priority text on_crash') def __init__(self, config): - self.messages = queue.PriorityQueue() # type: queue.PriorityQueue[Reporter._msg_type] + self.messages: queue.PriorityQueue[Reporter._msg_type] = queue.PriorityQueue() self.config = config def add_message(self, msg, priority, on_crash=True): diff --git a/certbot/certbot/_internal/snap_config.py b/certbot/certbot/_internal/snap_config.py index a90740787..f7832cd55 100644 --- a/certbot/certbot/_internal/snap_config.py +++ b/certbot/certbot/_internal/snap_config.py @@ -1,13 +1,13 @@ """Module configuring Certbot in a snap environment""" import logging import socket +from typing import List from requests import Session from requests.adapters import HTTPAdapter from requests.exceptions import HTTPError from requests.exceptions import RequestException -from acme.magic_typing import List from certbot.compat import os from certbot.errors import Error @@ -33,8 +33,7 @@ _ARCH_TRIPLET_MAP = { LOGGER = logging.getLogger(__name__) -def prepare_env(cli_args): - # type: (List[str]) -> List[str] +def prepare_env(cli_args: List[str]) -> List[str]: """ Prepare runtime environment for a certbot execution in snap. :param list cli_args: List of command line arguments @@ -81,7 +80,7 @@ def prepare_env(cli_args): class _SnapdConnection(HTTPConnection): def __init__(self): - super(_SnapdConnection, self).__init__("localhost") + super().__init__("localhost") self.sock = None def connect(self): @@ -91,7 +90,7 @@ class _SnapdConnection(HTTPConnection): class _SnapdConnectionPool(HTTPConnectionPool): def __init__(self): - super(_SnapdConnectionPool, self).__init__("localhost") + super().__init__("localhost") def _new_conn(self): return _SnapdConnection() diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 690567a17..11dae33e9 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -5,13 +5,14 @@ import logging import re import shutil import stat +from typing import Optional import configobj -import parsedatetime -import pytz from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.hazmat.primitives.serialization import load_pem_private_key +import parsedatetime +import pytz import certbot from certbot import crypto_util @@ -58,7 +59,7 @@ def renewal_file_for_certname(config, certname): return path -def cert_path_for_cert_name(config, cert_name): +def cert_path_for_cert_name(config: interfaces.IConfig, cert_name: str) -> str: """ If `--cert-name` was specified, but you need a value for `--cert-path`. :param `configuration.NamespaceConfig` config: parsed command line arguments @@ -66,10 +67,7 @@ def cert_path_for_cert_name(config, cert_name): """ cert_name_implied_conf = renewal_file_for_certname(config, cert_name) - fullchain_path = configobj.ConfigObj(cert_name_implied_conf)["fullchain"] - with open(fullchain_path) as f: - cert_path = (fullchain_path, f.read()) - return cert_path + return configobj.ConfigObj(cert_name_implied_conf)["fullchain"] def config_with_defaults(config=None): @@ -521,11 +519,15 @@ class RenewableCert(interfaces.RenewableCert): return _relpath_from_file(self.archive_dir, from_file) @property - def is_test_cert(self): + def server(self) -> Optional[str]: + """Returns the ACME server associated with this certificate""" + return self.configuration["renewalparams"].get("server", None) + + @property + def is_test_cert(self) -> bool: """Returns true if this is a test cert from a staging server.""" - server = self.configuration["renewalparams"].get("server", None) - if server: - return util.is_staging(server) + if self.server: + return util.is_staging(self.server) return False def _check_symlinks(self): diff --git a/certbot/certbot/achallenges.py b/certbot/certbot/achallenges.py index 7171c271c..a3ddbdcd3 100644 --- a/certbot/certbot/achallenges.py +++ b/certbot/certbot/achallenges.py @@ -18,15 +18,16 @@ Note, that all annotated challenges act as a proxy objects:: """ import logging +from typing import Type import josepy as jose from acme import challenges +from acme.challenges import Challenge logger = logging.getLogger(__name__) - class AnnotatedChallenge(jose.ImmutableMap): """Client annotated challenge. @@ -37,7 +38,7 @@ class AnnotatedChallenge(jose.ImmutableMap): """ __slots__ = ('challb',) - acme_type = NotImplemented + _acme_type: Type[Challenge] = NotImplemented def __getattr__(self, name): return getattr(self.challb, name) diff --git a/certbot/certbot/compat/_path.py b/certbot/certbot/compat/_path.py index 44c5e300f..a5db3a575 100644 --- a/certbot/certbot/compat/_path.py +++ b/certbot/certbot/compat/_path.py @@ -3,6 +3,10 @@ This compat module wraps os.path to forbid some functions. isort:skip_file """ + +# NB: Each function defined in compat._path is marked with "type: ignore" to avoid mypy +# to complain that a function is redefined (because we imported if first from os.path). + # pylint: disable=function-redefined from __future__ import absolute_import @@ -29,7 +33,7 @@ del ourselves, std_os_path, std_sys # Function os.path.realpath is broken on some versions of Python for Windows. -def realpath(*unused_args, **unused_kwargs): +def realpath(*unused_args, **unused_kwargs): # type: ignore """Method os.path.realpath() is forbidden""" raise RuntimeError('Usage of os.path.realpath() is forbidden. ' 'Use certbot.compat.filesystem.realpath() instead.') diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index 0152685e9..25e1687da 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -5,8 +5,7 @@ import errno import os # pylint: disable=os-module-forbidden import stat import sys - -from acme.magic_typing import List +from typing import List try: import ntsecuritycon @@ -36,8 +35,7 @@ class _WindowsUmask: _WINDOWS_UMASK = _WindowsUmask() -def chmod(file_path, mode): - # type: (str, int) -> None +def chmod(file_path: str, mode: int) -> None: """ Apply a POSIX mode on given file_path: @@ -58,8 +56,7 @@ def chmod(file_path, mode): _apply_win_mode(file_path, mode) -def umask(mask): - # type: (int) -> int +def umask(mask: int) -> int: """ Set the current numeric umask and return the previous umask. On Linux, the built-in umask method is used. On Windows, our Certbot-side implementation is used. @@ -85,8 +82,8 @@ def umask(mask): # Since copying and editing arbitrary DACL is very difficult, and since we actually know # the mode to apply at the time the owner of a file should change, it is easier to just # change the owner, then reapply the known mode, as copy_ownership_and_apply_mode() does. -def copy_ownership_and_apply_mode(src, dst, mode, copy_user, copy_group): - # type: (str, str, int, bool, bool) -> None +def copy_ownership_and_apply_mode(src: str, dst: str, mode: int, + copy_user: bool, copy_group: bool) -> None: """ Copy ownership (user and optionally group on Linux) from the source to the destination, then apply given mode in compatible way for Linux and Windows. @@ -118,8 +115,8 @@ def copy_ownership_and_apply_mode(src, dst, mode, copy_user, copy_group): # equivalent POSIX mode, because ownership and mode are copied altogether on the destination # file, so no recomputing of the DACL against the new owner is needed, as it would be # for a copy_ownership alone method. -def copy_ownership_and_mode(src, dst, copy_user=True, copy_group=True): - # type: (str, str, bool, bool) -> None +def copy_ownership_and_mode(src: str, dst: str, + copy_user: bool = True, copy_group: bool = True) -> None: """ Copy ownership (user and optionally group on Linux) and mode/DACL from the source to the destination. @@ -143,8 +140,7 @@ def copy_ownership_and_mode(src, dst, copy_user=True, copy_group=True): _copy_win_mode(src, dst) -def check_mode(file_path, mode): - # type: (str, int) -> bool +def check_mode(file_path: str, mode: int) -> bool: """ Check if the given mode matches the permissions of the given file. On Linux, will make a direct comparison, on Windows, mode will be compared against @@ -161,8 +157,7 @@ def check_mode(file_path, mode): return _check_win_mode(file_path, mode) -def check_owner(file_path): - # type: (str) -> bool +def check_owner(file_path: str) -> bool: """ Check if given file is owned by current user. @@ -184,8 +179,7 @@ def check_owner(file_path): return _get_current_user() == user -def check_permissions(file_path, mode): - # type: (str, int) -> bool +def check_permissions(file_path: str, mode: int) -> bool: """ Check if given file has the given mode and is owned by current user. @@ -197,8 +191,7 @@ def check_permissions(file_path, mode): return check_owner(file_path) and check_mode(file_path, mode) -def open(file_path, flags, mode=0o777): # pylint: disable=redefined-builtin - # type: (str, int, int) -> int +def open(file_path: str, flags: int, mode: int = 0o777) -> int: # pylint: disable=redefined-builtin """ Wrapper of original os.open function, that will ensure on Windows that given mode is correctly applied. @@ -267,8 +260,7 @@ def open(file_path, flags, mode=0o777): # pylint: disable=redefined-builtin return handle -def makedirs(file_path, mode=0o777): - # type: (str, int) -> None +def makedirs(file_path: str, mode: int = 0o777) -> None: """ Rewrite of original os.makedirs function, that will ensure on Windows that given mode is correctly applied. @@ -300,8 +292,7 @@ def makedirs(file_path, mode=0o777): umask(current_umask) -def mkdir(file_path, mode=0o777): - # type: (str, int) -> None +def mkdir(file_path: str, mode: int = 0o777) -> None: """ Rewrite of original os.mkdir function, that will ensure on Windows that given mode is correctly applied. @@ -332,8 +323,7 @@ def mkdir(file_path, mode=0o777): return None -def replace(src, dst): - # type: (str, str) -> None +def replace(src: str, dst: str) -> None: """ Rename a file to a destination path and handles situations where the destination exists. @@ -350,8 +340,7 @@ def replace(src, dst): os.rename(src, dst) -def realpath(file_path): - # type: (str) -> str +def realpath(file_path: str) -> str: """ Find the real path for the given path. This method resolves symlinks, including recursive symlinks, and is protected against symlinks that creates an infinite loop. @@ -372,7 +361,7 @@ def realpath(file_path): raise RuntimeError('Error, link {0} is a loop!'.format(original_path)) return path - inspected_paths = [] # type: List[str] + inspected_paths: List[str] = [] while os.path.islink(file_path): link_path = file_path file_path = os.readlink(file_path) @@ -385,8 +374,7 @@ def realpath(file_path): return os.path.abspath(file_path) -def readlink(link_path): - # type: (str) -> str +def readlink(link_path: str) -> str: """ Return a string representing the path to which the symbolic link points. @@ -420,8 +408,7 @@ def readlink(link_path): # elevated privileges or not. However this is not a problem since certbot always # requires to be run under a privileged shell, so the user will always benefit # from the highest (privileged one) set of permissions on a given file. -def is_executable(path): - # type: (str) -> bool +def is_executable(path: str) -> bool: """ Is path an executable file? @@ -435,8 +422,7 @@ def is_executable(path): return _win_is_executable(path) -def has_world_permissions(path): - # type: (str) -> bool +def has_world_permissions(path: str) -> bool: """ Check if everybody/world has any right (read/write/execute) on a file given its path. @@ -457,8 +443,7 @@ def has_world_permissions(path): })) -def compute_private_key_mode(old_key, base_mode): - # type: (str, int) -> int +def compute_private_key_mode(old_key: str, base_mode: int) -> int: """ Calculate the POSIX mode to apply to a private key given the previous private key. @@ -479,8 +464,7 @@ def compute_private_key_mode(old_key, base_mode): return base_mode -def has_same_ownership(path1, path2): - # type: (str, str) -> bool +def has_same_ownership(path1: str, path2: str) -> bool: """ Return True if the ownership of two files given their respective path is the same. On Windows, ownership is checked against owner only, since files do not have a group owner. @@ -505,8 +489,7 @@ def has_same_ownership(path1, path2): return user1 == user2 -def has_min_permissions(path, min_mode): - # type: (str, int) -> bool +def has_min_permissions(path: str, min_mode: int) -> bool: """ Check if a file given its path has at least the permissions defined by the given minimal mode. On Windows, group permissions are ignored since files do not have a group owner. diff --git a/certbot/certbot/compat/misc.py b/certbot/certbot/compat/misc.py index f4ea4a5cc..3d2624906 100644 --- a/certbot/certbot/compat/misc.py +++ b/certbot/certbot/compat/misc.py @@ -8,12 +8,12 @@ import logging import select import subprocess import sys +from typing import Optional +from typing import Tuple from certbot import errors from certbot.compat import os -from acme.magic_typing import Tuple, Optional - try: from win32com.shell import shell as shellwin32 POSIX_MODE = False @@ -27,8 +27,7 @@ logger = logging.getLogger(__name__) STANDARD_BINARY_DIRS = ["/usr/sbin", "/usr/local/bin", "/usr/local/sbin"] if POSIX_MODE else [] -def raise_for_non_administrative_windows_rights(): - # type: () -> None +def raise_for_non_administrative_windows_rights() -> None: """ On Windows, raise if current shell does not have the administrative rights. Do nothing on Linux. @@ -39,8 +38,7 @@ def raise_for_non_administrative_windows_rights(): raise errors.Error('Error, certbot must be run on a shell with administrative rights.') -def readline_with_timeout(timeout, prompt): - # type: (float, str) -> str +def readline_with_timeout(timeout: float, prompt: str) -> str: """ Read user input to return the first line entered, or raise after specified timeout. @@ -81,8 +79,7 @@ LINUX_DEFAULT_FOLDERS = { } -def get_default_folder(folder_type): - # type: (str) -> str +def get_default_folder(folder_type: str) -> str: """ Return the relevant default folder for the current OS @@ -99,8 +96,7 @@ def get_default_folder(folder_type): return WINDOWS_DEFAULT_FOLDERS[folder_type] -def underscores_for_unsupported_characters_in_path(path): - # type: (str) -> str +def underscores_for_unsupported_characters_in_path(path: str) -> str: """ Replace unsupported characters in path for current OS by underscores. :param str path: the path to normalize @@ -116,8 +112,7 @@ def underscores_for_unsupported_characters_in_path(path): return drive + tail.replace(':', '_') -def execute_command(cmd_name, shell_cmd, env=None): - # type: (str, str, Optional[dict]) -> Tuple[str, str] +def execute_command(cmd_name: str, shell_cmd: str, env: Optional[dict] = None) -> Tuple[str, str]: """ Run a command: - on Linux command will be run by the standard shell selected with Popen(shell=True) diff --git a/certbot/certbot/compat/os.py b/certbot/certbot/compat/os.py index 2f0899bd7..653877b50 100644 --- a/certbot/certbot/compat/os.py +++ b/certbot/certbot/compat/os.py @@ -6,12 +6,15 @@ This module is intended to replace standard os module throughout certbot project This module has the same API as the os module in the Python standard library except for the functions defined below. +isort:skip_file """ -# NOTE: If adding a new documented function to compat.os, ensure that it is added to the +# NB1: If adding a new documented function to compat.os, ensure that it is added to the # ':members:' list in certbot/docs/api/certbot.compat.os.rst. -# isort:skip_file +# NB2: Each function defined in compat.os is marked with "type: ignore" to avoid mypy +# to complain that a function is redefined (because we imported if first from os). + # pylint: disable=function-redefined from __future__ import absolute_import @@ -60,7 +63,7 @@ del ourselves, std_os, std_sys # Basically, it states that appropriate permissions will be set for the owner, nothing for the # group, appropriate permissions for the "Everyone" group, and all permissions to the # "Administrators" group + "System" user, as they can do everything anyway. -def chmod(*unused_args, **unused_kwargs): +def chmod(*unused_args, **unused_kwargs): # type: ignore """Method os.chmod() is forbidden""" raise RuntimeError('Usage of os.chmod() is forbidden. ' 'Use certbot.compat.filesystem.chmod() instead.') @@ -70,7 +73,7 @@ def chmod(*unused_args, **unused_kwargs): # this platform. In order to have a consistent behavior between Linux and Windows on Certbot files # and directories, the filesystem umask method must be used instead, since it implements umask for # Windows. -def umask(*unused_args, **unused_kwargs): +def umask(*unused_args, **unused_kwargs): # type: ignore """Method os.chmod() is forbidden""" raise RuntimeError('Usage of os.umask() is forbidden. ' 'Use certbot.compat.filesystem.umask() instead.') @@ -79,7 +82,7 @@ def umask(*unused_args, **unused_kwargs): # Because uid is not a concept on Windows, chown is useless. In fact, it is not even available # on Python for Windows. So to be consistent on both platforms for Certbot, this method is # always forbidden. -def chown(*unused_args, **unused_kwargs): +def chown(*unused_args, **unused_kwargs): # type: ignore """Method os.chown() is forbidden""" raise RuntimeError('Usage of os.chown() is forbidden.' 'Use certbot.compat.filesystem.copy_ownership_and_apply_mode() instead.') @@ -90,7 +93,7 @@ def chown(*unused_args, **unused_kwargs): # filesystem.open invokes the Windows native API `CreateFile` to ensure that permissions are # atomically set in case of file creation, or invokes filesystem.chmod to properly set the # permissions for the other cases. -def open(*unused_args, **unused_kwargs): +def open(*unused_args, **unused_kwargs): # type: ignore """Method os.open() is forbidden""" raise RuntimeError('Usage of os.open() is forbidden. ' 'Use certbot.compat.filesystem.open() instead.') @@ -98,7 +101,7 @@ def open(*unused_args, **unused_kwargs): # Very similarly to os.open, os.mkdir has the same effects on Windows and creates an unsecured # folder. So a similar mitigation to security.chmod is provided on this platform. -def mkdir(*unused_args, **unused_kwargs): +def mkdir(*unused_args, **unused_kwargs): # type: ignore """Method os.mkdir() is forbidden""" raise RuntimeError('Usage of os.mkdir() is forbidden. ' 'Use certbot.compat.filesystem.mkdir() instead.') @@ -109,7 +112,7 @@ def mkdir(*unused_args, **unused_kwargs): # that our modified os.mkdir is called on Windows, by monkey patching temporarily the mkdir method # on the original os module, executing the modified logic to correctly protect newly created # folders, then restoring original mkdir method in the os module. -def makedirs(*unused_args, **unused_kwargs): +def makedirs(*unused_args, **unused_kwargs): # type: ignore """Method os.makedirs() is forbidden""" raise RuntimeError('Usage of os.makedirs() is forbidden. ' 'Use certbot.compat.filesystem.makedirs() instead.') @@ -117,7 +120,7 @@ def makedirs(*unused_args, **unused_kwargs): # Because of the blocking strategy on file handlers on Windows, rename does not behave as expected # with POSIX systems: an exception will be raised if dst already exists. -def rename(*unused_args, **unused_kwargs): +def rename(*unused_args, **unused_kwargs): # type: ignore """Method os.rename() is forbidden""" raise RuntimeError('Usage of os.rename() is forbidden. ' 'Use certbot.compat.filesystem.replace() instead.') @@ -125,7 +128,7 @@ def rename(*unused_args, **unused_kwargs): # Behavior of os.replace is consistent between Windows and Linux. However, it is not supported on # Python 2.x. So, as for os.rename, we forbid it in favor of filesystem.replace. -def replace(*unused_args, **unused_kwargs): +def replace(*unused_args, **unused_kwargs): # type: ignore """Method os.replace() is forbidden""" raise RuntimeError('Usage of os.replace() is forbidden. ' 'Use certbot.compat.filesystem.replace() instead.') @@ -133,7 +136,7 @@ def replace(*unused_args, **unused_kwargs): # Results given by os.access are inconsistent or partial on Windows, because this platform is not # following the POSIX approach. -def access(*unused_args, **unused_kwargs): +def access(*unused_args, **unused_kwargs): # type: ignore """Method os.access() is forbidden""" raise RuntimeError('Usage of os.access() is forbidden. ' 'Use certbot.compat.filesystem.check_mode() or ' @@ -142,7 +145,7 @@ def access(*unused_args, **unused_kwargs): # On Windows os.stat call result is inconsistent, with a lot of flags that are not set or # meaningless. We need to use specialized functions from the certbot.compat.filesystem module. -def stat(*unused_args, **unused_kwargs): +def stat(*unused_args, **unused_kwargs): # type: ignore """Method os.stat() is forbidden""" raise RuntimeError('Usage of os.stat() is forbidden. ' 'Use certbot.compat.filesystem functions instead ' @@ -151,7 +154,7 @@ def stat(*unused_args, **unused_kwargs): # Method os.fstat has the same problem than os.stat, since it is the same function, # but accepting a file descriptor instead of a path. -def fstat(*unused_args, **unused_kwargs): +def fstat(*unused_args, **unused_kwargs): # type: ignore """Method os.stat() is forbidden""" raise RuntimeError('Usage of os.fstat() is forbidden. ' 'Use certbot.compat.filesystem functions instead ' @@ -163,7 +166,7 @@ def fstat(*unused_args, **unused_kwargs): # unconditionally, which allows to use more than 259 characters, and its string # representation is prepended with "\\?\". Problem is that it does it for any path, # and will make equality comparison fail with paths that will use the simple form. -def readlink(*unused_args, **unused_kwargs): +def readlink(*unused_args, **unused_kwargs): # type: ignore """Method os.readlink() is forbidden""" raise RuntimeError('Usage of os.readlink() is forbidden. ' 'Use certbot.compat.filesystem.realpath() instead.') diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index f67d95d97..5592722dd 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -6,26 +6,28 @@ """ import hashlib import logging +import re import warnings -import re # See https://github.com/pyca/cryptography/issues/4275 from cryptography import x509 # type: ignore -from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm +from cryptography.exceptions import InvalidSignature +from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import ec -from cryptography.hazmat.primitives.asymmetric.ec import ECDSA, EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey -from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat +from cryptography.hazmat.primitives.serialization import Encoding +from cryptography.hazmat.primitives.serialization import NoEncryption +from cryptography.hazmat.primitives.serialization import PrivateFormat from OpenSSL import crypto from OpenSSL import SSL # type: ignore - import pyrfc3339 import zope.component from acme import crypto_util as acme_crypto_util -from acme.magic_typing import IO # pylint: disable=unused-import from certbot import errors from certbot import interfaces from certbot import util @@ -269,9 +271,9 @@ def verify_renewable_cert_sig(renewable_cert): :raises errors.Error: If signature verification fails. """ try: - with open(renewable_cert.chain_path, 'rb') as chain_file: # type: IO[bytes] + with open(renewable_cert.chain_path, 'rb') as chain_file: chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) - with open(renewable_cert.cert_path, 'rb') as cert_file: # type: IO[bytes] + with open(renewable_cert.cert_path, 'rb') as cert_file: cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) pk = chain.public_key() with warnings.catch_warnings(): @@ -299,8 +301,7 @@ def verify_signed_payload(public_key, signature, payload, signature_hash_algorit with warnings.catch_warnings(): warnings.simplefilter("ignore") if isinstance(public_key, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = public_key.verifier( # type: ignore + verifier = public_key.verifier( signature, PKCS1v15(), signature_hash_algorithm ) verifier.update(payload) @@ -346,11 +347,11 @@ def verify_fullchain(renewable_cert): :raises errors.Error: If cert and chain do not combine to fullchain. """ try: - with open(renewable_cert.chain_path) as chain_file: # type: IO[str] + with open(renewable_cert.chain_path) as chain_file: chain = chain_file.read() - with open(renewable_cert.cert_path) as cert_file: # type: IO[str] + with open(renewable_cert.cert_path) as cert_file: cert = cert_file.read() - with open(renewable_cert.fullchain_path) as fullchain_file: # type: IO[str] + with open(renewable_cert.fullchain_path) as fullchain_file: fullchain = fullchain_file.read() if (cert + chain) != fullchain: error_str = "fullchain does not match cert + chain for {0}!" @@ -484,7 +485,7 @@ def _notAfterBefore(cert_path, method): """ # pylint: disable=redefined-outer-name - with open(cert_path, "rb") as f: # type: IO[bytes] + with open(cert_path, "rb") as f: x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) # pyopenssl always returns bytes timestamp = method(x509) @@ -561,7 +562,7 @@ def get_serial_from_cert(cert_path): :rtype: int """ # pylint: disable=redefined-outer-name - with open(cert_path, "rb") as f: # type: IO[bytes] + with open(cert_path, "rb") as f: x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) return x509.get_serial_number() diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index 9da981892..dc642586c 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -12,11 +12,11 @@ Other messages can use the `logging` module. See `log.py`. import logging import sys import textwrap +from typing import List -import zope.interface import zope.component +import zope.interface -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot._internal import constants @@ -98,8 +98,7 @@ def input_with_timeout(prompt=None, timeout=36000.0): return line.rstrip('\n') -def notify(msg): - # type: (str) -> None +def notify(msg: str) -> None: """Display a basic status message. :param str msg: message to display @@ -111,12 +110,12 @@ def notify(msg): @zope.interface.implementer(interfaces.IDisplay) -class FileDisplay(object): +class FileDisplay: """File-based display.""" # see https://github.com/certbot/certbot/issues/3915 def __init__(self, outfile, force_interactive): - super(FileDisplay, self).__init__() + super().__init__() self.outfile = outfile self.force_interactive = force_interactive self.skipped_interaction = False @@ -478,11 +477,11 @@ def assert_valid_call(prompt, default, cli_flag, force_interactive): @zope.interface.implementer(interfaces.IDisplay) -class NoninteractiveDisplay(object): +class NoninteractiveDisplay: """An iDisplay implementation that never asks for interactive user input""" def __init__(self, outfile, *unused_args, **unused_kwargs): - super(NoninteractiveDisplay, self).__init__() + super().__init__() self.outfile = outfile def _interaction_fail(self, message, cli_flag, extra=""): @@ -636,8 +635,7 @@ def _parens_around_char(label): return "({first}){rest}".format(first=label[0], rest=label[1:]) -def summarize_domain_list(domains): - # type: (List[str]) -> str +def summarize_domain_list(domains: List[str]) -> str: """Summarizes a list of domains in the format of: example.com.com and N more domains or if there is are only two domains: diff --git a/certbot/certbot/errors.py b/certbot/certbot/errors.py index 48aebc267..cf0d3d283 100644 --- a/certbot/certbot/errors.py +++ b/certbot/certbot/errors.py @@ -53,7 +53,7 @@ class FailedChallenges(AuthorizationError): def __init__(self, failed_achalls): assert failed_achalls self.failed_achalls = failed_achalls - super(FailedChallenges, self).__init__() + super().__init__() def __str__(self): return "Failed authorization procedure. {0}".format( @@ -95,7 +95,7 @@ class StandaloneBindError(Error): """Standalone plugin bind error.""" def __init__(self, socket_error, port): - super(StandaloneBindError, self).__init__( + super().__init__( "Problem binding to port {0}: {1}".format(port, socket_error)) self.socket_error = socket_error self.port = port diff --git a/certbot/certbot/interfaces.py b/certbot/certbot/interfaces.py index ddbee8ddc..de9175def 100644 --- a/certbot/certbot/interfaces.py +++ b/certbot/certbot/interfaces.py @@ -1,5 +1,6 @@ """Certbot client interfaces.""" import abc +from typing import Optional import zope.interface @@ -327,7 +328,7 @@ class IInstaller(IPlugin): """ - def save(title=None, temporary=False): + def save(title: Optional[str] = None, temporary: bool = False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be @@ -349,7 +350,7 @@ class IInstaller(IPlugin): """ - def rollback_checkpoints(rollback=1): + def rollback_checkpoints(rollback: int = 1): """Revert `rollback` number of configuration checkpoints. :raises .PluginError: when configuration cannot be fully reverted diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index b63338e2e..0a842e108 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -5,6 +5,8 @@ import logging import re from subprocess import PIPE from subprocess import Popen +from typing import Optional +from typing import Tuple from cryptography import x509 from cryptography.exceptions import InvalidSignature @@ -16,8 +18,6 @@ from cryptography.hazmat.primitives import serialization import pytz import requests -from acme.magic_typing import Optional -from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import util @@ -36,7 +36,7 @@ except (ImportError, AttributeError): # pragma: no cover logger = logging.getLogger(__name__) -class RevocationChecker(object): +class RevocationChecker: """This class figures out OCSP checking on this system, and performs it.""" def __init__(self, enforce_openssl_binary_usage=False): @@ -59,8 +59,7 @@ class RevocationChecker(object): else: self.host_args = lambda host: ["Host", host] - def ocsp_revoked(self, cert): - # type: (RenewableCert) -> bool + def ocsp_revoked(self, cert: RenewableCert) -> bool: """Get revoked status for a particular cert version. .. todo:: Make this a non-blocking call @@ -72,8 +71,7 @@ class RevocationChecker(object): """ return self.ocsp_revoked_by_paths(cert.cert_path, cert.chain_path) - def ocsp_revoked_by_paths(self, cert_path, chain_path, timeout=10): - # type: (str, str, int) -> bool + def ocsp_revoked_by_paths(self, cert_path: str, chain_path: str, timeout: int = 10) -> bool: """Performs the OCSP revocation check :param str cert_path: Certificate filepath @@ -102,8 +100,8 @@ class RevocationChecker(object): return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url, timeout) return _check_ocsp_cryptography(cert_path, chain_path, url, timeout) - def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url, timeout): - # type: (str, str, str, str, int) -> bool + def _check_ocsp_openssl_bin(self, cert_path: str, chain_path: str, + host: str, url: str, timeout: int) -> bool: # Minimal implementation of proxy selection logic as seen in, e.g., cURL # Some things that won't work, but may well be in use somewhere: # - username and password for proxy authentication @@ -140,8 +138,7 @@ class RevocationChecker(object): return _translate_ocsp_query(cert_path, output, err) -def _determine_ocsp_server(cert_path): - # type: (str) -> Tuple[Optional[str], Optional[str]] +def _determine_ocsp_server(cert_path: str) -> Tuple[Optional[str], Optional[str]]: """Extract the OCSP server host from a certificate. :param str cert_path: Path to the cert we're checking OCSP for @@ -171,8 +168,7 @@ def _determine_ocsp_server(cert_path): return None, None -def _check_ocsp_cryptography(cert_path, chain_path, url, timeout): - # type: (str, str, str, int) -> bool +def _check_ocsp_cryptography(cert_path: str, chain_path: str, url: str, timeout: int) -> bool: # Retrieve OCSP response with open(chain_path, 'rb') as file_handler: issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index d3fb5b7dc..4c33acbab 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -3,13 +3,13 @@ import logging import re import shutil import tempfile +from typing import List from josepy import util as jose_util import pkg_resources import zope.interface -from acme.magic_typing import List -from certbot import achallenges # pylint: disable=unused-import +from certbot import achallenges from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -40,7 +40,7 @@ hostname_regex = re.compile( @zope.interface.implementer(interfaces.IPlugin) -class Plugin(object): +class Plugin: """Generic plugin.""" # provider is not inherited, subclasses must define it on their own # @zope.interface.provider(interfaces.IPluginFactory) @@ -105,7 +105,7 @@ class Installer(Plugin): """ def __init__(self, *args, **kwargs): - super(Installer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.storage = PluginStorage(self.config, self.name) self.reverter = reverter.Reverter(self.config) @@ -201,7 +201,7 @@ class Installer(Plugin): constants.ALL_SSL_DHPARAMS_HASHES) -class Addr(object): +class Addr: r"""Represents an virtual host address. :param str addr: addr part of vhost address @@ -299,7 +299,7 @@ class Addr(object): return result -class ChallengePerformer(object): +class ChallengePerformer: """Abstract base for challenge performers. :ivar configurator: Authenticator and installer plugin @@ -313,8 +313,8 @@ class ChallengePerformer(object): def __init__(self, configurator): self.configurator = configurator - self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] - self.indices = [] # type: List[int] + self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = [] + self.indices: List[int] = [] def add_chall(self, achall, idx=None): """Store challenge to be performed when perform() is called. diff --git a/certbot/certbot/plugins/dns_common.py b/certbot/certbot/plugins/dns_common.py index 245b7dc05..4459ac0fa 100644 --- a/certbot/certbot/plugins/dns_common.py +++ b/certbot/certbot/plugins/dns_common.py @@ -25,7 +25,7 @@ class DNSAuthenticator(common.Plugin): """Base class for DNS Authenticators""" def __init__(self, config, name): - super(DNSAuthenticator, self).__init__(config, name) + super().__init__(config, name) self._attempt_cleanup = False @@ -43,6 +43,9 @@ class DNSAuthenticator(common.Plugin): def prepare(self): # pylint: disable=missing-function-docstring pass + def more_info(self) -> str: # pylint: disable=missing-function-docstring + raise NotImplementedError() + def perform(self, achalls): # pylint: disable=missing-function-docstring self._setup_credentials() @@ -139,7 +142,8 @@ class DNSAuthenticator(common.Plugin): setattr(self.config, self.dest(key), os.path.abspath(os.path.expanduser(new_value))) - def _configure_credentials(self, key, label, required_variables=None, validator=None): + def _configure_credentials(self, key, label, required_variables=None, + validator=None) -> 'CredentialsConfiguration': """ As `_configure_file`, but for a credential configuration file. @@ -233,7 +237,7 @@ class DNSAuthenticator(common.Plugin): raise errors.PluginError('{0} required to proceed.'.format(label)) -class CredentialsConfiguration(object): +class CredentialsConfiguration: """Represents a user-supplied filed which stores API credentials.""" def __init__(self, filename, mapper=lambda x: x): diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index c3d80ca29..a4d46587e 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -1,12 +1,12 @@ """Common code for DNS Authenticator Plugins built on Lexicon.""" import logging +from typing import Any +from typing import Dict +from typing import Union from requests.exceptions import HTTPError from requests.exceptions import RequestException -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import Union from certbot import errors from certbot.plugins import dns_common @@ -17,19 +17,21 @@ from certbot.plugins import dns_common # if Lexicon is not available, obviously. try: from lexicon.config import ConfigResolver + from lexicon.providers.base import Provider except ImportError: ConfigResolver = None # type: ignore + Provider = None # type: ignore logger = logging.getLogger(__name__) -class LexiconClient(object): +class LexiconClient: """ Encapsulates all communication with a DNS provider via Lexicon. """ def __init__(self): - self.provider = None + self.provider: Provider def add_txt_record(self, domain, record_name, record_content): """ @@ -116,8 +118,9 @@ class LexiconClient(object): return None -def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_options): - # type: (str, Dict, Dict) -> Union[ConfigResolver, Dict] +def build_lexicon_config(lexicon_provider_name: str, + lexicon_options: Dict, provider_options: Dict + ) -> Union[ConfigResolver, Dict]: """ Convenient function to build a Lexicon 2.x/3.x config object. :param str lexicon_provider_name: the name of the lexicon provider to use @@ -126,7 +129,7 @@ def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_option :return: configuration to apply to the provider :rtype: ConfigurationResolver or dict """ - config = {'provider_name': lexicon_provider_name} # type: Dict[str, Any] + config: Dict[str, Any] = {'provider_name': lexicon_provider_name} config.update(lexicon_options) if not ConfigResolver: # Lexicon 2.x diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index a3a26a7a1..7a8df9329 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -1,23 +1,53 @@ """Base test class for DNS authenticators.""" +import typing import configobj import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore from acme import challenges from certbot import achallenges from certbot.compat import filesystem +from certbot.plugins.dns_common import DNSAuthenticator from certbot.tests import acme_util from certbot.tests import util as test_util +if typing.TYPE_CHECKING: + from typing_extensions import Protocol +else: + Protocol = object # type: ignore + + + +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore + + DOMAIN = 'example.com' KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) -class BaseAuthenticatorTest(object): +class _AuthenticatorCallableTestCase(Protocol): + """Protocol describing a TestCase able to call a real DNSAuthenticator instance.""" + auth: DNSAuthenticator + + def assertTrue(self, *unused_args) -> None: + """ + See + https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertTrue + """ + ... + + def assertEqual(self, *unused_args) -> None: + """ + See + https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertEqual + """ + ... + + +class BaseAuthenticatorTest: """ A base test class to reduce duplication between test code for DNS Authenticator Plugins. @@ -29,13 +59,13 @@ class BaseAuthenticatorTest(object): achall = achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.DNS01, domain=DOMAIN, account_key=KEY) - def test_more_info(self): + def test_more_info(self: _AuthenticatorCallableTestCase): self.assertTrue(isinstance(self.auth.more_info(), str)) # pylint: disable=no-member - def test_get_chall_pref(self): + def test_get_chall_pref(self: _AuthenticatorCallableTestCase): self.assertEqual(self.auth.get_chall_pref(None), [challenges.DNS01]) # pylint: disable=no-member - def test_parser_arguments(self): + def test_parser_arguments(self: _AuthenticatorCallableTestCase): m = mock.MagicMock() self.auth.add_parser_arguments(m) # pylint: disable=no-member diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index 1bef06042..5c6f09d20 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -1,32 +1,79 @@ """Base test class for DNS authenticators built on Lexicon.""" +import typing +from unittest.mock import MagicMock import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException +from acme.challenges import Challenge from certbot import errors from certbot.plugins import dns_test_common +from certbot.plugins.dns_common_lexicon import LexiconClient +from certbot.plugins.dns_test_common import _AuthenticatorCallableTestCase from certbot.tests import util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore +if typing.TYPE_CHECKING: + from typing_extensions import Protocol +else: + Protocol = object # type: ignore + + + + DOMAIN = 'example.com' KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) + +class _AuthenticatorCallableLexiconTestCase(_AuthenticatorCallableTestCase, Protocol): + """ + Protocol describing a TestCase suitable to test challenges against + a mocked LexiconClient instance. + """ + mock_client: MagicMock + achall: Challenge + + +class _LexiconAwareTestCase(Protocol): + """ + Protocol describing a TestCase suitable to test a real LexiconClient instance. + """ + client: LexiconClient + provider_mock: MagicMock + + record_prefix: str + record_name: str + record_content: str + + DOMAIN_NOT_FOUND: Exception + GENERIC_ERROR: Exception + LOGIN_ERROR: Exception + UNKNOWN_LOGIN_ERROR: Exception + + def assertRaises(self, *unused_args) -> None: + """ + See + https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertRaises + """ + ... + + # These classes are intended to be subclassed/mixed in, so not all members are defined. # pylint: disable=no-member class BaseLexiconAuthenticatorTest(dns_test_common.BaseAuthenticatorTest): - def test_perform(self): + def test_perform(self: _AuthenticatorCallableLexiconTestCase): self.auth.perform([self.achall]) expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] self.assertEqual(expected, self.mock_client.mock_calls) - def test_cleanup(self): + def test_cleanup(self: _AuthenticatorCallableLexiconTestCase): self.auth._attempt_cleanup = True # _attempt_cleanup | pylint: disable=protected-access self.auth.cleanup([self.achall]) @@ -34,7 +81,7 @@ class BaseLexiconAuthenticatorTest(dns_test_common.BaseAuthenticatorTest): self.assertEqual(expected, self.mock_client.mock_calls) -class BaseLexiconClientTest(object): +class BaseLexiconClientTest: DOMAIN_NOT_FOUND = Exception('No domain found') GENERIC_ERROR = RequestException LOGIN_ERROR = HTTPError('400 Client Error: ...') @@ -44,14 +91,14 @@ class BaseLexiconClientTest(object): record_name = record_prefix + "." + DOMAIN record_content = "bar" - def test_add_txt_record(self): + def test_add_txt_record(self: _LexiconAwareTestCase): self.client.add_txt_record(DOMAIN, self.record_name, self.record_content) self.provider_mock.create_record.assert_called_with(type='TXT', name=self.record_name, content=self.record_content) - def test_add_txt_record_try_twice_to_find_domain(self): + def test_add_txt_record_try_twice_to_find_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, ''] self.client.add_txt_record(DOMAIN, self.record_name, self.record_content) @@ -60,7 +107,7 @@ class BaseLexiconClientTest(object): name=self.record_name, content=self.record_content) - def test_add_txt_record_fail_to_find_domain(self): + def test_add_txt_record_fail_to_find_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND,] @@ -69,64 +116,64 @@ class BaseLexiconClientTest(object): self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_fail_to_authenticate(self): + def test_add_txt_record_fail_to_authenticate(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.LOGIN_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_fail_to_authenticate_with_unknown_error(self): + def test_add_txt_record_fail_to_authenticate_with_unknown_error(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.UNKNOWN_LOGIN_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_error_finding_domain(self): + def test_add_txt_record_error_finding_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.GENERIC_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_error_adding_record(self): + def test_add_txt_record_error_adding_record(self: _LexiconAwareTestCase): self.provider_mock.create_record.side_effect = self.GENERIC_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_del_txt_record(self): + def test_del_txt_record(self: _LexiconAwareTestCase): self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) self.provider_mock.delete_record.assert_called_with(type='TXT', name=self.record_name, content=self.record_content) - def test_del_txt_record_fail_to_find_domain(self): + def test_del_txt_record_fail_to_find_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND, ] self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_fail_to_authenticate(self): + def test_del_txt_record_fail_to_authenticate(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.LOGIN_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_fail_to_authenticate_with_unknown_error(self): + def test_del_txt_record_fail_to_authenticate_with_unknown_error(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.UNKNOWN_LOGIN_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_error_finding_domain(self): + def test_del_txt_record_error_finding_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.GENERIC_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_error_deleting_record(self): + def test_del_txt_record_error_deleting_record(self: _LexiconAwareTestCase): self.provider_mock.delete_record.side_effect = self.GENERIC_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) diff --git a/certbot/certbot/plugins/enhancements.py b/certbot/certbot/plugins/enhancements.py index e674c32a2..c1de4d44f 100644 --- a/certbot/certbot/plugins/enhancements.py +++ b/certbot/certbot/plugins/enhancements.py @@ -1,9 +1,9 @@ """New interface style Certbot enhancements""" import abc +from typing import Any +from typing import Dict +from typing import List -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot._internal import constants ENHANCEMENTS = ["redirect", "ensure-http-header", "ocsp-stapling"] @@ -156,7 +156,7 @@ class AutoHSTSEnhancement(object, metaclass=abc.ABCMeta): # This is used to configure internal new style enhancements in Certbot. These # enhancement interfaces need to be defined in this file. Please do not modify # this list from plugin code. -_INDEX = [ +_INDEX: List[Dict[str, Any]] = [ { "name": "AutoHSTS", "cli_help": "Gradually increasing max-age value for HTTP Strict Transport "+ @@ -171,4 +171,4 @@ _INDEX = [ "deployer_function": "deploy_autohsts", "enable_function": "enable_autohsts" } -] # type: List[Dict[str, Any]] +] diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index f3ed14dce..602c62d1e 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -1,9 +1,9 @@ """Plugin storage class.""" import json import logging +from typing import Any +from typing import Dict -from acme.magic_typing import Any -from acme.magic_typing import Dict from certbot import errors from certbot.compat import filesystem from certbot.compat import os @@ -11,7 +11,7 @@ from certbot.compat import os logger = logging.getLogger(__name__) -class PluginStorage(object): +class PluginStorage: """Class implementing storage functionality for plugins""" def __init__(self, config, classkey): @@ -26,8 +26,8 @@ class PluginStorage(object): self._config = config self._classkey = classkey self._initialized = False - self._data = None - self._storagepath = None + self._data: Dict + self._storagepath: str def _initialize_storage(self): """Initializes PluginStorage data and reads current state from the disk @@ -42,7 +42,7 @@ class PluginStorage(object): :raises .errors.PluginStorageError: when unable to open or read the file """ - data = {} # type: Dict[str, Any] + data: Dict[str, Any] = {} filedata = "" try: with open(self._storagepath, 'r') as fh: diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index be9d78a11..ebc69fcbe 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -3,11 +3,9 @@ import csv import glob import logging import shutil -import sys import time import traceback - from certbot import errors from certbot import util from certbot._internal import constants @@ -17,7 +15,7 @@ from certbot.compat import os logger = logging.getLogger(__name__) -class Reverter(object): +class Reverter: """Reverter Class - save and revert configuration checkpoints. This class can be used by the plugins, especially Installers, to @@ -251,11 +249,10 @@ class Reverter(object): def _run_undo_commands(self, filepath): """Run all commands in a file.""" - # NOTE: csv module uses native strings. That is, bytes on Python 2 and - # unicode on Python 3 + # NOTE: csv module uses native strings. That is unicode on Python 3 # It is strongly advised to set newline = '' on Python 3 with CSV, # and it fixes problems on Windows. - kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + kwargs = {'newline': ''} with open(filepath, 'r', **kwargs) as csvfile: # type: ignore csvreader = csv.reader(csvfile) for command in reversed(list(csvreader)): @@ -354,7 +351,7 @@ class Reverter(object): command_file = None # It is strongly advised to set newline = '' on Python 3 with CSV, # and it fixes problems on Windows. - kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + kwargs = {'newline': ''} try: if os.path.isfile(commands_fp): command_file = open(commands_fp, "a", **kwargs) # type: ignore diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 558bfbed3..b892ad0a1 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -27,6 +27,9 @@ from certbot.compat import os from certbot.display import util as display_util try: + # When we remove this deprecated import, we should also remove the + # "external-mock" test environment and the mock dependency listed in + # tools/pinning/pyproject.toml. import mock warnings.warn( "The external mock module is being used for backwards compatibility " @@ -185,7 +188,7 @@ def patch_get_utility_with_stdout(target='zope.component.getUtility', return mock.patch(target, new=freezable_mock) -class FreezableMock(object): +class FreezableMock: """Mock object with the ability to freeze attributes. This class works like a regular mock.MagicMock object, except @@ -331,7 +334,7 @@ class TempDirTestCase(unittest.TestCase): class ConfigTestCase(TempDirTestCase): """Test class which sets up a NamespaceConfig object.""" def setUp(self): - super(ConfigTestCase, self).setUp() + super().setUp() self.config = configuration.NamespaceConfig( mock.MagicMock(**constants.CLI_DEFAULTS) ) diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index 7bbc02f5f..5f4a08dc7 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -4,7 +4,6 @@ import argparse import atexit import collections -from collections import OrderedDict import distutils.version import errno import logging @@ -13,12 +12,13 @@ import re import socket import subprocess import sys +from typing import Dict +from typing import Text +from typing import Tuple +from typing import Union import configargparse -from acme.magic_typing import Text -from acme.magic_typing import Tuple -from acme.magic_typing import Union from certbot import errors from certbot._internal import constants from certbot._internal import lock @@ -58,7 +58,7 @@ _INITIAL_PID = os.getpid() # the dict are attempted to be cleaned up at program exit. If the # program exits before the lock is cleaned up, it is automatically # released, but the file isn't deleted. -_LOCKS = OrderedDict() # type: OrderedDict[str, lock.LockFile] +_LOCKS: Dict[str, lock.LockFile] = {} def env_no_snap_for_external_calls(): @@ -216,10 +216,10 @@ def safe_open(path, mode="w", chmod=None): if ``None``. """ - open_args = () # type: Union[Tuple[()], Tuple[int]] + open_args: Union[Tuple[()], Tuple[int]] = () if chmod is not None: open_args = (chmod,) - fdopen_args = () # type: Union[Tuple[()], Tuple[int]] + fdopen_args: Union[Tuple[()], Tuple[int]] = () fd = filesystem.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args) return os.fdopen(fd, mode, *fdopen_args) @@ -577,7 +577,7 @@ def is_wildcard_domain(domain): :rtype: bool """ - wildcard_marker = b"*." # type: Union[Text, bytes] + wildcard_marker: Union[Text, bytes] = b"*." if isinstance(domain, str): wildcard_marker = u"*." return domain.startswith(wildcard_marker) diff --git a/certbot/docs/ciphers.rst b/certbot/docs/ciphers.rst index 43f648898..b4ef5902a 100644 --- a/certbot/docs/ciphers.rst +++ b/certbot/docs/ciphers.rst @@ -1,3 +1,11 @@ +.. + Sphinx complains that this file isn't included in any toctree, however, we + currently link to it in the section about installing Certbot through Docker. + Setting :orphan: below suppresses this warning. See + https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html#special-metadata-fields. + +:orphan: + ============ Ciphersuites ============ diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index 4482ea439..5d772d90e 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -118,12 +118,12 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.12.0 - (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX - Installer/YYY (SUBCOMMAND; flags: FLAGS) - Py/major.minor.patchlevel). The flags encoded in the - user agent are: --duplicate, --force-renew, --allow- - subset-of-names, -n, and whether any hooks are set. + "". (default: CertbotACMEClient/1.15.0 (certbot; + OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). + The flags encoded in the user agent are: --duplicate, + --force-renew, --allow-subset-of-names, -n, and + whether any hooks are set. --user-agent-comment USER_AGENT_COMMENT Add a comment to the default user agent string. May be used when repackaging Certbot or calling it from @@ -167,19 +167,6 @@ automation: --duplicate Allow making a certificate lineage that duplicates an existing one (both can be renewed in parallel) (default: False) - --os-packages-only (certbot-auto only) install OS package dependencies - and then stop (default: False) - --no-self-upgrade (certbot-auto only) prevent the certbot-auto script - from upgrading itself to newer released versions - (default: Upgrade automatically) - --no-bootstrap (certbot-auto only) prevent the certbot-auto script - from installing OS-level dependencies (default: Prompt - to install OS-wide dependencies, but exit if the user - says 'No') - --no-permissions-check - (certbot-auto only) skip the check on the file system - permissions of the certbot-auto script (default: - False) -q, --quiet Silence all output except errors. Useful for automation via cron. Implies --non-interactive. (default: False) @@ -229,9 +216,7 @@ testing: (invalid) certificates; equivalent to --server https://acme-staging-v02.api.letsencrypt.org/directory (default: False) - --debug Show tracebacks in case of errors, and allow certbot- - auto execution on experimental platforms (default: - False) + --debug Show tracebacks in case of errors (default: False) --no-verify-ssl Disable verification of the ACME server's certificate. (default: False) --http-01-port HTTP01_PORT @@ -254,8 +239,8 @@ paths: Flags for changing execution paths & servers --cert-path CERT_PATH - Path to where certificate is saved (with auth --csr), - installed from, or revoked. (default: None) + Path to where certificate is saved (with certonly + --csr), installed from, or revoked (default: None) --key-path KEY_PATH Path to private key for certificate installation or revocation (if account key is missing) (default: None) --fullchain-path FULLCHAIN_PATH diff --git a/certbot/docs/compatibility.rst b/certbot/docs/compatibility.rst index a4f33c281..d94642ec6 100644 --- a/certbot/docs/compatibility.rst +++ b/certbot/docs/compatibility.rst @@ -21,9 +21,9 @@ may change at any time. The second is that Certbot's behavior should only be considered stable with certain files but not all. Files with which users should expect Certbot to maintain its current behavior with are: -* ``/etc/letsencrypt/live//{cert,chain,fullchain,privkey}.pem`` where - ```` is the name given to ``--cert-name``. If ``--cert-name`` is not - set by the user, it is the first domain given to ``--domains``. +* ``/etc/letsencrypt/live/$domain/{cert,chain,fullchain,privkey}.pem``, where + ``$domain`` is the certificate name (see :ref:`where-certs` + for more details) * :ref:`CLI configuration files ` * Hook directories in ``/etc/letsencrypt/renewal-hooks`` diff --git a/certbot/docs/conf.py b/certbot/docs/conf.py index 254bd3edd..5496c42a2 100644 --- a/certbot/docs/conf.py +++ b/certbot/docs/conf.py @@ -98,7 +98,6 @@ language = None exclude_patterns = [ '_build', 'challenges.rst', - 'ciphers.rst' ] # The reST default role (used for this markup: `text`) to use for all diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index def2c7fcd..94e35f1bf 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -222,8 +222,6 @@ certbot-apache and certbot-nginx client code to configure specific web servers certbot-dns-* client code to configure DNS providers -certbot-auto and letsencrypt-auto - shell scripts to install Certbot and its dependencies on UNIX systems windows installer Installs Certbot on Windows and is built using the files in windows-installer/ @@ -478,13 +476,6 @@ to start contributing to Certbot. To run mypy on Certbot, use ``tox -e mypy`` on a machine that has Python 3 installed. -Note that instead of just importing ``typing``, due to packaging issues, in Certbot we import from -``acme.magic_typing`` and have to add some comments for pylint like this: - -.. code-block:: python - - from acme.magic_typing import Dict - Also note that OpenSSL, which we rely on, has type definitions for crypto but not SSL. We use both. Those imports should look like this: @@ -555,53 +546,6 @@ Instructions for how to manually build and run the Certbot snap and the external snapped DNS plugins that the Certbot project supplies are located in the README file at https://github.com/certbot/certbot/tree/master/tools/snap. -Updating certbot-auto and letsencrypt-auto -========================================== - -.. note:: We are currently only accepting changes to certbot-auto that fix - regressions on platforms where certbot-auto is the recommended installation - method at https://certbot.eff.org/instructions. If you are unsure if a change - you want to make qualifies, don't hesitate to `ask for help`_! - -Updating the scripts --------------------- -Developers should *not* modify the ``certbot-auto`` and ``letsencrypt-auto`` files -in the root directory of the repository. Rather, modify the -``letsencrypt-auto.template`` and associated platform-specific shell scripts in -the ``letsencrypt-auto-source`` and -``letsencrypt-auto-source/pieces/bootstrappers`` directory, respectively. - -Building letsencrypt-auto-source/letsencrypt-auto -------------------------------------------------- -Once changes to any of the aforementioned files have been made, the -``letsencrypt-auto-source/letsencrypt-auto`` script should be updated. In lieu of -manually updating this script, run the build script, which lives at -``letsencrypt-auto-source/build.py``: - -.. code-block:: shell - - python letsencrypt-auto-source/build.py - -Running ``build.py`` will update the ``letsencrypt-auto-source/letsencrypt-auto`` -script. Note that the ``certbot-auto`` and ``letsencrypt-auto`` scripts in the root -directory of the repository will remain **unchanged** after this script is run. -Your changes will be propagated to these files during the next release of -Certbot. - -Opening a PR ------------- -When opening a PR, ensure that the following files are committed: - -1. ``letsencrypt-auto-source/letsencrypt-auto.template`` and - ``letsencrypt-auto-source/pieces/bootstrappers/*`` -2. ``letsencrypt-auto-source/letsencrypt-auto`` (generated by ``build.py``) - -It might also be a good idea to double check that **no** changes were -inadvertently made to the ``certbot-auto`` or ``letsencrypt-auto`` scripts in the -root of the repository. These scripts will be updated by the core developers -during the next release. - - Updating the documentation ========================== diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index 8ae1c82f2..4533cfcc1 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -125,117 +125,6 @@ of the ``/etc/letsencrypt`` directory, see :ref:`where-certs`. .. _Docker: https://docker.com .. _`install Docker`: https://docs.docker.com/engine/installation/ -Operating System Packages -------------------------- - -.. warning:: While the Certbot team tries to keep the Certbot packages offered - by various operating systems working in the most basic sense, due to - distribution policies and/or the limited resources of distribution - maintainers, Certbot OS packages often have problems that other distribution - mechanisms do not. The packages are often old resulting in a lack of bug - fixes and features and a worse TLS configuration than is generated by newer - versions of Certbot. They also may not configure certificate renewal for you - or have all of Certbot's plugins available. For reasons like these, we - recommend most users follow the instructions at - https://certbot.eff.org/instructions and OS packages are only documented - here as an alternative. - -**Arch Linux** - -.. code-block:: shell - - sudo pacman -S certbot - -**Debian** - -If you run Debian Buster or Debian testing/Sid, you can easily install certbot -packages through commands like: - -.. code-block:: shell - - sudo apt-get update - sudo apt-get install certbot - -If you run Debian Stretch, we recommend you use the packages in Debian -backports repository. First you'll have to follow the instructions at -https://backports.debian.org/Instructions/ to enable the Stretch backports repo, -if you have not already done so. Then run: - -.. code-block:: shell - - sudo apt-get install certbot -t stretch-backports - -In all of these cases, there also packages available to help Certbot integrate -with Apache, nginx, or various DNS services. If you are using Apache or nginx, -we strongly recommend that you install the ``python-certbot-apache`` or -``python-certbot-nginx`` package so that Certbot can fully automate HTTPS -configuration for your server. A full list of these packages can be found -through a command like: - -.. code-block:: shell - - apt search 'python-certbot*' - -They can be installed by running the same installation command above but -replacing ``certbot`` with the name of the desired package. - -**Ubuntu** - -If you run Ubuntu, certbot can be installed using: - -.. code-block:: shell - - sudo apt-get install certbot - -Optionally to install the Certbot Apache plugin, you can use: - -.. code-block:: shell - - sudo apt-get install python3-certbot-apache - -**Fedora** - -.. code-block:: shell - - sudo dnf install certbot python3-certbot-apache - -**FreeBSD** - - * Port: ``cd /usr/ports/security/py-certbot && make install clean`` - * Package: ``pkg install py37-certbot`` - -**Gentoo** - -The official Certbot client is available in Gentoo Portage. From the -official Certbot plugins, three of them are also available in Portage. -They need to be installed separately if you require their functionality. - -.. code-block:: shell - - emerge -av app-crypt/certbot - emerge -av app-crypt/certbot-apache - emerge -av app-crypt/certbot-nginx - emerge -av app-crypt/certbot-dns-nsone - -.. Note:: The ``app-crypt/certbot-dns-nsone`` package has a different - maintainer than the other packages and can lag behind in version. - -**NetBSD** - - * Build from source: ``cd /usr/pkgsrc/security/py-certbot && make install clean`` - * Install pre-compiled package: ``pkg_add py37-certbot`` - -**OpenBSD** - - * Port: ``cd /usr/ports/security/letsencrypt/client && make install clean`` - * Package: ``pkg_add letsencrypt`` - -**Other Operating Systems** - -OS packaging is an ongoing effort. If you'd like to package -Certbot for your distribution of choice please have a -look at the :doc:`packaging`. - .. _certbot-auto: Certbot-Auto @@ -251,34 +140,10 @@ Certbot on UNIX operating systems, however, this script is no longer supported. If you want to uninstall ``certbot-auto``, you can follow our instructions :doc:`here `. -Problems with Python virtual environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Pip +--- -When using ``certbot-auto`` on a low memory system such as VPS with less than -512MB of RAM, the required dependencies of Certbot may fail to build. This can -be identified if the pip outputs contains something like ``internal compiler -error: Killed (program cc1)``. You can workaround this restriction by creating -a temporary swapfile:: - - user@webserver:~$ sudo fallocate -l 1G /tmp/swapfile - user@webserver:~$ sudo chmod 600 /tmp/swapfile - user@webserver:~$ sudo mkswap /tmp/swapfile - user@webserver:~$ sudo swapon /tmp/swapfile - -Disable and remove the swapfile once the virtual environment is constructed:: - - user@webserver:~$ sudo swapoff /tmp/swapfile - user@webserver:~$ sudo rm /tmp/swapfile - -Installing from source ----------------------- - -Installation from source is only supported for developers and the -whole process is described in the :doc:`contributing`. - -.. warning:: Please do **not** use ``python certbot/setup.py install``, ``python pip - install certbot``, or ``easy_install certbot``. Please do **not** attempt the - installation commands as superuser/root and/or without virtual environment, - e.g. ``sudo python certbot/setup.py install``, ``sudo pip install``, ``sudo - ./venv/bin/...``. These modes of operation might corrupt your operating - system and are **not supported** by the Certbot team! +Installing Certbot through pip is only supported on a best effort basis and +when using a virtual environment. Instructions for installing Certbot through +pip can be found at https://certbot.eff.org/instructions by selecting your +server software and then choosing "pip" in the "System" dropdown menu. diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index ab8d64d79..cc061b622 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -14,7 +14,7 @@ obtaining, renewing, or revoking certificates. The most important and commonly-used commands will be discussed throughout this document; an exhaustive list also appears near the end of the document. -The ``certbot`` script on your web server might be named ``letsencrypt`` if your system uses an older package, or ``certbot-auto`` if you used an alternate installation method. Throughout the docs, whenever you see ``certbot``, swap in the correct name as needed. +The ``certbot`` script on your web server might be named ``letsencrypt`` if your system uses an older package, or ``certbot-auto`` if you used a now-deprecated installation method. Throughout the docs, whenever you see ``certbot``, swap in the correct name as needed. .. _plugins: @@ -284,6 +284,7 @@ dns-ispconfig_ Y N DNS Authentication using ISPConfig as DNS server dns-clouddns_ Y N DNS Authentication using CloudDNS API dns-lightsail_ Y N DNS Authentication using Amazon Lightsail DNS API dns-inwx_ Y Y DNS Authentication for INWX through the XML API +dns-azure_ Y N DNS Authentication using Azure DNS ================== ==== ==== =============================================================== .. _haproxy: https://github.com/greenhost/certbot-haproxy @@ -298,6 +299,7 @@ dns-inwx_ Y Y DNS Authentication for INWX through the XML API .. _dns-clouddns: https://github.com/vshosting/certbot-dns-clouddns .. _dns-lightsail: https://github.com/noi/certbot-dns-lightsail .. _dns-inwx: https://github.com/oGGy990/certbot-dns-inwx/ +.. _dns-azure: https://github.com/binkhq/certbot-dns-azure If you're interested, you can also :ref:`write your own plugin `. @@ -420,7 +422,7 @@ option to control the curve used in ECDSA certificates. .. warning:: If you obtain certificates using ECDSA keys, you should be careful not to downgrade your Certbot installation since ECDSA keys are not supported by older versions of Certbot. Downgrades like this are possible if - you switch from something like the snaps or certbot-auto to packages + you switch from something like the snaps or pip to packages provided by your operating system which often lag behind. Changing existing certificates from RSA to ECDSA @@ -474,29 +476,37 @@ like Revoking certificates --------------------- -If your account key has been compromised or you otherwise need to revoke a certificate, -use the ``revoke`` command to do so. Note that the ``revoke`` command takes the certificate path -(ending in ``cert.pem``), not a certificate name or domain. Example:: +If you need to revoke a certificate, use the ``revoke`` subcommand to do so. - certbot revoke --cert-path /etc/letsencrypt/live/CERTNAME/cert.pem +A certificate may be revoked by providing its name (see ``certbot certificates``) or by providing +its path directly:: + + certbot revoke --cert-name example.com + + certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem + +If the certificate being revoked was obtained via the ``--staging``, ``--test-cert`` or a non-default ``--server`` flag, +that flag must be passed to the ``revoke`` subcommand. + +.. note:: After revocation, Certbot will (by default) ask whether you want to **delete** the certificate. + Unless deleted, Certbot will try to renew revoked certificates the next time ``certbot renew`` runs. You can also specify the reason for revoking your certificate by using the ``reason`` flag. Reasons include ``unspecified`` which is the default, as well as ``keycompromise``, ``affiliationchanged``, ``superseded``, and ``cessationofoperation``:: - certbot revoke --cert-path /etc/letsencrypt/live/CERTNAME/cert.pem --reason keycompromise + certbot revoke --cert-name example.com --reason keycompromise -Additionally, if a certificate -is a test certificate obtained via the ``--staging`` or ``--test-cert`` flag, that flag must be passed to the -``revoke`` subcommand. -Once a certificate is revoked (or for other certificate management tasks), all of a certificate's -relevant files can be removed from the system with the ``delete`` subcommand:: +Revoking by account key or certificate private key +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - certbot delete --cert-name example.com +By default, Certbot will try revoke the certificate using your ACME account key. If the certificate was created from +the same ACME account, the revocation will be successful. -.. note:: If you don't use ``delete`` to remove the certificate completely, it will be renewed automatically at the next renewal event. +If you instead have the corresponding private key file to the certificate you wish to revoke, use ``--key-path`` to perform the +revocation from any ACME account:: -.. note:: Revoking a certificate will have no effect on the rate limit imposed by the Let's Encrypt server. + certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem --key-path /etc/letsencrypt/live/example.com/privkey.pem .. _renewal: @@ -507,11 +517,8 @@ Renewing certificates days). Make sure you renew the certificates at least once in 3 months. -.. seealso:: Many of the certbot clients obtained through a - distribution come with automatic renewal out of the box, - such as Debian and Ubuntu versions installed through `apt`, - CentOS/RHEL 7 through EPEL, etc. See `Automated Renewals`_ - for more details. +.. seealso:: Most Certbot installations come with automatic + renewal out of the box. See `Automated Renewals`_ for more details. As of version 0.10.0, Certbot supports a ``renew`` action to check all installed certificates for impending expiry and attempt to renew @@ -681,27 +688,15 @@ The following commands could be used to specify where these files are located:: Automated Renewals ------------------ -Many Linux distributions provide automated renewal when you use the -packages installed through their system package manager. The -following table is an *incomplete* list of distributions which do so, -as well as their methods for doing so. +Most Certbot installations come with automatic renewals preconfigured. This +is done by means of a scheduled task which runs ``certbot renew`` periodically. -If you are not sure whether or not your system has this already -automated, refer to your distribution's documentation, or check your -system's crontab (typically in `/etc/crontab/` and `/etc/cron.*/*` and -systemd timers (`systemctl list-timers`). +If you are unsure whether you need to configure automated renewal: -.. csv-table:: Distributions with Automated Renewal - :header: "Distribution Name", "Distribution Version", "Automation Method" - - "CentOS", "EPEL 7", "systemd" - "Debian", "stretch", "cron, systemd" - "Debian", "testing/sid", "cron, systemd" - "Fedora", "26", "systemd" - "Fedora", "27", "systemd" - "RHEL", "EPEL 7", "systemd" - "Ubuntu", "17.10", "cron, systemd" - "Ubuntu", "certbot PPA", "cron, systemd" +1. Review the instructions for your system at https://certbot.eff.org/instructions. + They will describe how to set up a scheduled task, if necessary. +2. (Linux/BSD): Check your system's crontab (typically `/etc/crontab` and + `/etc/cron.*/*`) and systemd timers (``systemctl list-timers``). .. _where-certs: @@ -709,12 +704,24 @@ Where are my certificates? ========================== All generated keys and issued certificates can be found in -``/etc/letsencrypt/live/$domain``. In the case of creating a SAN certificate -with multiple alternative names, ``$domain`` is the first domain passed in -via -d parameter. Rather than copying, please point -your (web) server configuration directly to those files (or create -symlinks). During the renewal_, ``/etc/letsencrypt/live`` is updated -with the latest necessary files. +``/etc/letsencrypt/live/$domain``, where ``$domain`` is the certificate +name (see the note below). Rather than copying, please point your (web) +server configuration directly to those files (or create symlinks). +During the renewal_, ``/etc/letsencrypt/live`` is updated with the latest +necessary files. + +.. note:: + The certificate name ``$domain`` used in the path ``/etc/letsencrypt/live/$domain`` + follows this convention: + + * it is the name given to ``--cert-name``, + * if ``--cert-name`` is not set by the user it is the first domain given to + ``--domains``, + * if the first domain is a wildcard domain (eg. ``*.example.com``) the + certificate name will be ``example.com``, + * if a name collision would occur with a certificate already named ``example.com``, + the new certificate name will be constructed using a numerical sequence + as ``example.com-001``. For historical reasons, the containing directories are created with permissions of ``0700`` meaning that certificates are accessible only diff --git a/certbot/setup.py b/certbot/setup.py index 8b2874f20..e0078bd6e 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -8,6 +8,12 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup +min_setuptools_version='39.0.1' +# This conditional isn't necessary, but it provides better error messages to +# people who try to install this package with older versions of setuptools. +if LooseVersion(setuptools_version) < LooseVersion(min_setuptools_version): + raise RuntimeError(f'setuptools {min_setuptools_version}+ is required') + # Workaround for https://bugs.python.org/issue8876, see # https://bugs.python.org/issue8876#msg208792 # This can be removed when using Python 2.7.9 or later: @@ -49,29 +55,14 @@ install_requires = [ 'parsedatetime>=2.4', 'pyrfc3339', 'pytz', - 'setuptools>=39.0.1', + # This dependency needs to be added using environment markers to avoid its + # installation on Linux. + 'pywin32>=300 ; sys_platform == "win32"', + f'setuptools>={min_setuptools_version}', 'zope.component', 'zope.interface', ] -# Add pywin32 on Windows platforms to handle low-level system calls. -# This dependency needs to be added using environment markers to avoid its installation on Linux. -# However environment markers are supported only with setuptools >= 36.2. -# So this dependency is not added for old Linux distributions with old setuptools, -# in order to allow these systems to build certbot from sources. -pywin32_req = 'pywin32>=300' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append(pywin32_req + " ; sys_platform == 'win32'") -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif os.name == 'nt': - # This branch exists to improve this package's behavior on Windows. Without - # it, if the sdist is installed on Windows with an old version of - # setuptools, pywin32 will not be specified as a dependency. - install_requires.append(pywin32_req) - dev_extras = [ 'astroid', 'azure-devops', @@ -79,10 +70,16 @@ dev_extras = [ 'ipdb', 'mypy', 'PyGithub', + # 1.1.0+ is required for poetry to use the poetry-core library for the + # build system declared in tools/pinning/pyproject.toml. + 'poetry>=1.1.0', 'pylint', 'pytest', 'pytest-cov', 'pytest-xdist', + # typing-extensions is required to import typing.Protocol and make the mypy checks + # pass (along with pylint about non-existent objects) on Python 3.6 & 3.7 + 'typing-extensions', 'tox', 'twine', 'wheel', @@ -103,7 +100,7 @@ setup( long_description=readme, url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index d0448a1db..e034c5f32 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -106,7 +106,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): """Tests for certbot._internal.account.AccountFileStorage.""" def setUp(self): - super(AccountFileStorageTest, self).setUp() + super().setUp() from certbot._internal.account import AccountFileStorage self.storage = AccountFileStorage(self.config) @@ -150,7 +150,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): path = os.path.join(self.config.accounts_dir, self.acc.id, "regr.json") with open(path, "r") as f: regr = json.load(f) - self.assertTrue("new_authzr_uri" in regr) + self.assertIn("new_authzr_uri", regr) def test_update_regr(self): self.storage.update_regr(self.acc, self.mock_client) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 8eb5d7702..1f798c2d8 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -106,9 +106,9 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_time.sleep.call_count, 2) # Retry-After header is 30 seconds, but at the time sleep is invoked, several # instructions are executed, and next pool is in less than 30 seconds. - self.assertTrue(mock_time.sleep.call_args_list[1][0][0] <= 30) + self.assertLessEqual(mock_time.sleep.call_args_list[1][0][0], 30) # However, assert that we did not took the default value of 3 seconds. - self.assertTrue(mock_time.sleep.call_args_list[1][0][0] > 3) + self.assertGreater(mock_time.sleep.call_args_list[1][0][0], 3) self.assertEqual(self.mock_auth.cleanup.call_count, 1) # Test if list first element is http-01, use typ because it is an achall @@ -139,7 +139,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_auth.cleanup.call_count, 1) # Test if list first element is http-01, use typ because it is an achall for achall in self.mock_auth.cleanup.call_args[0][0]: - self.assertTrue(achall.typ in ["http-01", "dns-01"]) + self.assertIn(achall.typ, ["http-01", "dns-01"]) # Length of authorizations list self.assertEqual(len(authzr), 1) @@ -225,7 +225,7 @@ class HandleAuthorizationsTest(unittest.TestCase): with self.assertRaises(errors.AuthorizationError) as error: # We retry only once, so retries will be exhausted before STATUS_VALID is returned. self.handler.handle_authorizations(mock_order, False, 1) - self.assertTrue('All authorizations were not finalized by the CA.' in str(error.exception)) + self.assertIn('All authorizations were not finalized by the CA.', str(error.exception)) def test_no_domains(self): mock_order = mock.MagicMock(authorizations=[]) @@ -305,7 +305,7 @@ class HandleAuthorizationsTest(unittest.TestCase): with test_util.patch_get_utility(): with self.assertRaises(errors.AuthorizationError) as error: self.handler.handle_authorizations(mock_order, False) - self.assertTrue('Some challenges have failed.' in str(error.exception)) + self.assertIn('Some challenges have failed.', str(error.exception)) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") @@ -341,7 +341,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.handler.handle_authorizations(mock_order, True) # Despite best_effort=True, process will fail because no authzr is valid. - self.assertTrue('All challenges have failed.' in str(error.exception)) + self.assertIn('All challenges have failed.', str(error.exception)) def test_validated_challenge_not_rerun(self): # With a pending challenge that is not supported by the plugin, we @@ -486,7 +486,7 @@ class ReportFailedAuthzrsTest(unittest.TestCase): } # Prevent future regressions if the error type changes - self.assertTrue(kwargs["error"].description is not None) + self.assertIsNotNone(kwargs["error"].description) http_01 = messages.ChallengeBody(**kwargs) @@ -511,7 +511,7 @@ class ReportFailedAuthzrsTest(unittest.TestCase): auth_handler._report_failed_authzrs([self.authzr1], 'key') call_list = mock_zope().add_message.call_args_list self.assertEqual(len(call_list), 1) - self.assertTrue("Domain: example.com\nType: tls\nDetail: detail" in call_list[0][0][0]) + self.assertIn("Domain: example.com\nType: tls\nDetail: detail", call_list[0][0][0]) @test_util.patch_get_utility() def test_different_errors_and_domains(self, mock_zope): diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index b26c1f624..3e8fb0de7 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -26,7 +26,7 @@ class BaseCertManagerTest(test_util.ConfigTestCase): """Base class for setting up Cert Manager tests. """ def setUp(self): - super(BaseCertManagerTest, self).setUp() + super().setUp() self.config.quiet = False filesystem.makedirs(self.config.renewal_configs_dir) @@ -211,7 +211,7 @@ class CertificatesTest(BaseCertManagerTest): def test_certificates_quiet(self, mock_utility, mock_logger): self.config.quiet = True self._certificates(self.config) - self.assertFalse(mock_utility.notification.called) + self.assertIs(mock_utility.notification.called, False) self.assertTrue(mock_logger.warning.called) #pylint: disable=no-member @mock.patch('certbot.crypto_util.verify_renewable_cert') @@ -224,7 +224,7 @@ class CertificatesTest(BaseCertManagerTest): mock_verifier.return_value = None mock_report.return_value = "" self._certificates(self.config) - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) self.assertTrue(mock_report.called) self.assertTrue(mock_utility.called) self.assertTrue(mock_renewable_cert.called) @@ -242,7 +242,7 @@ class CertificatesTest(BaseCertManagerTest): filesystem.makedirs(empty_config.renewal_configs_dir) self._certificates(empty_config) - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) self.assertTrue(mock_utility.called) shutil.rmtree(empty_tempdir) @@ -269,31 +269,34 @@ class CertificatesTest(BaseCertManagerTest): get_report = lambda: cert_manager._report_human_readable(mock_config, parsed_certs) out = get_report() - self.assertTrue("INVALID: EXPIRED" in out) + self.assertIn("INVALID: EXPIRED", out) cert.target_expiry += datetime.timedelta(hours=2) # pylint: disable=protected-access out = get_report() - self.assertTrue('1 hour(s)' in out or '2 hour(s)' in out) - self.assertTrue('VALID' in out and 'INVALID' not in out) + self.assertIs('1 hour' in out or '2 hour(s)' in out, True) + self.assertIn('VALID', out) + self.assertNotIn('INVALID', out) cert.target_expiry += datetime.timedelta(days=1) # pylint: disable=protected-access out = get_report() - self.assertTrue('1 day' in out) - self.assertFalse('under' in out) - self.assertTrue('VALID' in out and 'INVALID' not in out) + self.assertIn('1 day', out) + self.assertNotIn('under', out) + self.assertIn('VALID', out) + self.assertNotIn('INVALID', out) cert.target_expiry += datetime.timedelta(days=2) # pylint: disable=protected-access out = get_report() - self.assertTrue('3 days' in out) - self.assertTrue('VALID' in out and 'INVALID' not in out) + self.assertIn('3 days', out) + self.assertIn('VALID', out) + self.assertNotIn('INVALID', out) cert.is_test_cert = True mock_revoked.return_value = True out = get_report() - self.assertTrue('INVALID: TEST_CERT, REVOKED' in out) + self.assertIn('INVALID: TEST_CERT, REVOKED', out) cert = mock.MagicMock(lineagename="indescribable") cert.target_expiry = expiry @@ -353,7 +356,7 @@ class LineageForCertnameTest(BaseCertManagerTest): def test_no_match(self, mock_renewal_conf_file, mock_make_or_verify_dir): mock_renewal_conf_file.return_value = "other.com.conf" from certbot._internal import cert_manager - self.assertEqual(cert_manager.lineage_for_certname(self.config, "example.com"), None) + self.assertIsNone(cert_manager.lineage_for_certname(self.config, "example.com")) self.assertTrue(mock_make_or_verify_dir.called) @mock.patch('certbot.util.make_or_verify_dir') @@ -361,7 +364,7 @@ class LineageForCertnameTest(BaseCertManagerTest): def test_no_renewal_file(self, mock_renewal_conf_file, mock_make_or_verify_dir): mock_renewal_conf_file.side_effect = errors.CertStorageError() from certbot._internal import cert_manager - self.assertEqual(cert_manager.lineage_for_certname(self.config, "example.com"), None) + self.assertIsNone(cert_manager.lineage_for_certname(self.config, "example.com")) self.assertTrue(mock_make_or_verify_dir.called) @@ -388,7 +391,7 @@ class DomainsForCertnameTest(BaseCertManagerTest): def test_no_match(self, mock_renewal_conf_file, mock_make_or_verify_dir): mock_renewal_conf_file.return_value = "somefile.conf" from certbot._internal import cert_manager - self.assertEqual(cert_manager.domains_for_certname(self.config, "other.com"), None) + self.assertIsNone(cert_manager.domains_for_certname(self.config, "other.com")) self.assertTrue(mock_make_or_verify_dir.called) @@ -396,7 +399,7 @@ class RenameLineageTest(BaseCertManagerTest): """Tests for certbot._internal.cert_manager.rename_lineage""" def setUp(self): - super(RenameLineageTest, self).setUp() + super().setUp() self.config.certname = "example.org" self.config.new_certname = "after" @@ -450,7 +453,7 @@ class RenameLineageTest(BaseCertManagerTest): self._call(self.config) from certbot._internal import cert_manager updated_lineage = cert_manager.lineage_for_certname(self.config, self.config.new_certname) - self.assertTrue(updated_lineage is not None) + self.assertIsNotNone(updated_lineage) self.assertEqual(updated_lineage.lineagename, self.config.new_certname) @test_util.patch_get_utility() @@ -463,7 +466,7 @@ class RenameLineageTest(BaseCertManagerTest): self._call(self.config) from certbot._internal import cert_manager updated_lineage = cert_manager.lineage_for_certname(self.config, self.config.new_certname) - self.assertTrue(updated_lineage is not None) + self.assertIsNotNone(updated_lineage) self.assertEqual(updated_lineage.lineagename, self.config.new_certname) @test_util.patch_get_utility() @@ -483,7 +486,7 @@ class DuplicativeCertsTest(storage_test.BaseRenewableCertTest): """Test to avoid duplicate lineages.""" def setUp(self): - super(DuplicativeCertsTest, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() @@ -503,12 +506,12 @@ class DuplicativeCertsTest(storage_test.BaseRenewableCertTest): result = find_duplicative_certs( self.config, ['example.com', 'www.example.com']) self.assertTrue(result[0].configfile.filename.endswith('example.org.conf')) - self.assertEqual(result[1], None) + self.assertIsNone(result[1]) # Superset result = find_duplicative_certs( self.config, ['example.com', 'www.example.com', 'something.new']) - self.assertEqual(result[0], None) + self.assertIsNone(result[0]) self.assertTrue(result[1].configfile.filename.endswith('example.org.conf')) # Partial overlap doesn't count @@ -521,12 +524,12 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): """Tests for certbot._internal.cert_manager.cert_path_to_lineage""" def setUp(self): - super(CertPathToLineageTest, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config): from certbot._internal.cert_manager import cert_path_to_lineage @@ -556,21 +559,21 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): mock_acceptable_matches.return_value = [lambda x: x.cert_path] test_cert_path = os.path.join(self.config.config_dir, 'live', 'example.org', 'cert.pem') - self.config.cert_path = (test_cert_path, '') + self.config.cert_path = test_cert_path self.assertEqual('example.org', self._call(self.config)) @mock.patch('certbot._internal.cert_manager._acceptable_matches') def test_options_archive_cert(self, mock_acceptable_matches): # Also this and the next test check that the regex of _archive_files is working. - self.config.cert_path = (os.path.join(self.config.config_dir, 'archive', 'example.org', - 'cert11.pem'), '') + self.config.cert_path = os.path.join(self.config.config_dir, 'archive', 'example.org', + 'cert11.pem') mock_acceptable_matches.return_value = [lambda x: self._archive_files(x, 'cert')] self.assertEqual('example.org', self._call(self.config)) @mock.patch('certbot._internal.cert_manager._acceptable_matches') def test_options_archive_fullchain(self, mock_acceptable_matches): - self.config.cert_path = (os.path.join(self.config.config_dir, 'archive', - 'example.org', 'fullchain11.pem'), '') + self.config.cert_path = os.path.join(self.config.config_dir, 'archive', + 'example.org', 'fullchain11.pem') mock_acceptable_matches.return_value = [lambda x: self._archive_files(x, 'fullchain')] self.assertEqual('example.org', self._call(self.config)) @@ -581,12 +584,12 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): archive dirs.""" # A test with real overlapping archive dirs can be found in tests/boulder_integration.sh def setUp(self): - super(MatchAndCheckOverlaps, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config, acceptable_matches, match_func, rv_func): from certbot._internal.cert_manager import match_and_check_overlaps @@ -595,7 +598,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): def test_basic_match(self): from certbot._internal.cert_manager import _acceptable_matches self.assertEqual(['example.org'], self._call(self.config, _acceptable_matches(), - lambda x: self.config.cert_path[0], lambda x: x.lineagename)) + lambda x: self.config.cert_path, lambda x: x.lineagename)) @mock.patch('certbot._internal.cert_manager._search_lineages') def test_no_matches(self, mock_search_lineages): @@ -629,8 +632,7 @@ class GetCertnameTest(unittest.TestCase): self.assertEqual( cert_manager.get_certnames( self.config, "verb", allow_multiple=False), ['example.com']) - self.assertTrue( - prompt in self.mock_get_utility().menu.call_args[0][0]) + self.assertIn(prompt, self.mock_get_utility().menu.call_args[0][0]) @mock.patch('certbot._internal.storage.renewal_conf_files') @mock.patch('certbot._internal.storage.lineagename_for_filename') @@ -671,8 +673,7 @@ class GetCertnameTest(unittest.TestCase): self.assertEqual( cert_manager.get_certnames( self.config, "verb", allow_multiple=True), ['example.com']) - self.assertTrue( - prompt in self.mock_get_utility().checklist.call_args[0][0]) + self.assertIn(prompt, self.mock_get_utility().checklist.call_args[0][0]) @mock.patch('certbot._internal.storage.renewal_conf_files') @mock.patch('certbot._internal.storage.lineagename_for_filename') diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index fca2b3e3e..8cab7a5b1 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -150,80 +150,79 @@ class ParseTest(unittest.TestCase): def test_help(self): self._help_output(['--help']) # assert SystemExit is raised here out = self._help_output(['--help', 'all']) - self.assertTrue("--configurator" in out) - self.assertTrue("how a certificate is deployed" in out) - self.assertTrue("--webroot-path" in out) - self.assertTrue("--text" not in out) - self.assertTrue("%s" not in out) - self.assertTrue("{0}" not in out) - self.assertTrue("--renew-hook" not in out) + self.assertIn("--configurator", out) + self.assertIn("how a certificate is deployed", out) + self.assertIn("--webroot-path", out) + self.assertNotIn("--text", out) + self.assertNotIn("%s", out) + self.assertNotIn("{0}", out) + self.assertNotIn("--renew-hook", out) out = self._help_output(['-h', 'nginx']) if "nginx" in PLUGINS: # may be false while building distributions without plugins - self.assertTrue("--nginx-ctl" in out) - self.assertTrue("--webroot-path" not in out) - self.assertTrue("--checkpoints" not in out) + self.assertIn("--nginx-ctl", out) + self.assertNotIn("--webroot-path", out) + self.assertNotIn("--checkpoints", out) out = self._help_output(['-h']) - self.assertTrue("letsencrypt-auto" not in out) # test cli.cli_command if "nginx" in PLUGINS: - self.assertTrue("Use the Nginx plugin" in out) + self.assertIn("Use the Nginx plugin", out) else: - self.assertTrue("(the certbot nginx plugin is not" in out) + self.assertIn("(the certbot nginx plugin is not", out) out = self._help_output(['--help', 'plugins']) - self.assertTrue("--webroot-path" not in out) - self.assertTrue("--prepare" in out) - self.assertTrue('"plugins" subcommand' in out) + self.assertNotIn("--webroot-path", out) + self.assertIn("--prepare", out) + self.assertIn('"plugins" subcommand', out) # test multiple topics out = self._help_output(['-h', 'renew']) - self.assertTrue("--keep" in out) + self.assertIn("--keep", out) out = self._help_output(['-h', 'automation']) - self.assertTrue("--keep" in out) + self.assertIn("--keep", out) out = self._help_output(['-h', 'revoke']) - self.assertTrue("--keep" not in out) + self.assertNotIn("--keep", out) out = self._help_output(['--help', 'install']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) out = self._help_output(['--help', 'revoke']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) - self.assertTrue("--reason" in out) - self.assertTrue("--delete-after-revoke" in out) - self.assertTrue("--no-delete-after-revoke" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) + self.assertIn("--reason", out) + self.assertIn("--delete-after-revoke", out) + self.assertIn("--no-delete-after-revoke", out) out = self._help_output(['-h', 'register']) - self.assertTrue("--cert-path" not in out) - self.assertTrue("--key-path" not in out) + self.assertNotIn("--cert-path", out) + self.assertNotIn("--key-path", out) out = self._help_output(['-h']) - self.assertTrue(cli.SHORT_USAGE in out) - self.assertTrue(cli.COMMAND_OVERVIEW[:100] in out) - self.assertTrue("%s" not in out) - self.assertTrue("{0}" not in out) + self.assertIn(cli.SHORT_USAGE, out) + self.assertIn(cli.COMMAND_OVERVIEW[:100], out) + self.assertNotIn("%s", out) + self.assertNotIn("{0}", out) def test_help_no_dashes(self): self._help_output(['help']) # assert SystemExit is raised here out = self._help_output(['help', 'all']) - self.assertTrue("--configurator" in out) - self.assertTrue("how a certificate is deployed" in out) - self.assertTrue("--webroot-path" in out) - self.assertTrue("--text" not in out) - self.assertTrue("%s" not in out) - self.assertTrue("{0}" not in out) + self.assertIn("--configurator", out) + self.assertIn("how a certificate is deployed", out) + self.assertIn("--webroot-path", out) + self.assertNotIn("--text", out) + self.assertNotIn("%s", out) + self.assertNotIn("{0}", out) out = self._help_output(['help', 'install']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) out = self._help_output(['help', 'revoke']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) def test_parse_domains(self): short_args = ['-d', 'example.com'] @@ -271,8 +270,8 @@ class ParseTest(unittest.TestCase): def test_must_staple_flag(self): short_args = ['--must-staple'] namespace = self.parse(short_args) - self.assertTrue(namespace.must_staple) - self.assertTrue(namespace.staple) + self.assertIs(namespace.must_staple, True) + self.assertIs(namespace.staple, True) def _check_server_conflict_message(self, parser_args, conflicting_args): try: @@ -281,31 +280,31 @@ class ParseTest(unittest.TestCase): "The following flags didn't conflict with " '--server: {0}'.format(', '.join(conflicting_args))) except errors.Error as error: - self.assertTrue('--server' in str(error)) + self.assertIn('--server', str(error)) for arg in conflicting_args: - self.assertTrue(arg in str(error)) + self.assertIn(arg, str(error)) def test_staging_flag(self): short_args = ['--staging'] namespace = self.parse(short_args) - self.assertTrue(namespace.staging) + self.assertIs(namespace.staging, True) self.assertEqual(namespace.server, constants.STAGING_URI) short_args += '--server example.com'.split() self._check_server_conflict_message(short_args, '--staging') def _assert_dry_run_flag_worked(self, namespace, existing_account): - self.assertTrue(namespace.dry_run) - self.assertTrue(namespace.break_my_certs) - self.assertTrue(namespace.staging) + self.assertIs(namespace.dry_run, True) + self.assertIs(namespace.break_my_certs, True) + self.assertIs(namespace.staging, True) self.assertEqual(namespace.server, constants.STAGING_URI) if existing_account: - self.assertTrue(namespace.tos) - self.assertTrue(namespace.register_unsafely_without_email) + self.assertIs(namespace.tos, True) + self.assertIs(namespace.register_unsafely_without_email, True) else: - self.assertFalse(namespace.tos) - self.assertFalse(namespace.register_unsafely_without_email) + self.assertIs(namespace.tos, False) + self.assertIs(namespace.register_unsafely_without_email, False) def test_dry_run_flag(self): config_dir = tempfile.mkdtemp() @@ -351,8 +350,8 @@ class ParseTest(unittest.TestCase): key_size_value = cli.flag_default(key_size_option) self.parse('--rsa-key-size {0}'.format(key_size_value).split()) - self.assertTrue(cli.option_was_set(key_size_option, key_size_value)) - self.assertTrue(cli.option_was_set('no_verify_ssl', True)) + self.assertIs(cli.option_was_set(key_size_option, key_size_value), True) + self.assertIs(cli.option_was_set('no_verify_ssl', True), True) config_dir_option = 'config_dir' self.assertFalse(cli.option_was_set( @@ -426,7 +425,7 @@ class ParseTest(unittest.TestCase): value = "foo" namespace = self.parse( ["--renew-hook", value, "--disable-hook-validation"]) - self.assertEqual(namespace.deploy_hook, None) + self.assertIsNone(namespace.deploy_hook) self.assertEqual(namespace.renew_hook, value) def test_max_log_backups_error(self): @@ -457,19 +456,19 @@ class ParseTest(unittest.TestCase): self.assertFalse(self.parse(["--no-directory-hooks"]).directory_hooks) def test_no_directory_hooks_unset(self): - self.assertTrue(self.parse([]).directory_hooks) + self.assertIs(self.parse([]).directory_hooks, True) def test_delete_after_revoke(self): namespace = self.parse(["--delete-after-revoke"]) - self.assertTrue(namespace.delete_after_revoke) + self.assertIs(namespace.delete_after_revoke, True) def test_delete_after_revoke_default(self): namespace = self.parse([]) - self.assertEqual(namespace.delete_after_revoke, None) + self.assertIsNone(namespace.delete_after_revoke) def test_no_delete_after_revoke(self): namespace = self.parse(["--no-delete-after-revoke"]) - self.assertFalse(namespace.delete_after_revoke) + self.assertIs(namespace.delete_after_revoke, False) def test_allow_subset_with_wildcard(self): self.assertRaises(errors.Error, self.parse, @@ -478,7 +477,7 @@ class ParseTest(unittest.TestCase): def test_route53_no_revert(self): for help_flag in ['-h', '--help']: for topic in ['all', 'plugins', 'dns-route53']: - self.assertFalse('certbot-route53:auth' in self._help_output([help_flag, topic])) + self.assertNotIn('certbot-route53:auth', self._help_output([help_flag, topic])) class DefaultTest(unittest.TestCase): @@ -491,8 +490,8 @@ class DefaultTest(unittest.TestCase): self.default2 = cli._Default() def test_boolean(self): - self.assertFalse(self.default1) - self.assertFalse(self.default2) + self.assertIs(bool(self.default1), False) + self.assertIs(bool(self.default2), False) def test_equality(self): self.assertEqual(self.default1, self.default2) @@ -515,7 +514,7 @@ class SetByCliTest(unittest.TestCase): def test_webroot_map(self): args = '-w /var/www/html -d example.com'.split() verb = 'renew' - self.assertTrue(_call_set_by_cli('webroot_map', args, verb)) + self.assertIs(_call_set_by_cli('webroot_map', args, verb), True) def _call_set_by_cli(var, args, verb): diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f058cb658..a89636a63 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -47,7 +47,6 @@ class DetermineUserAgentTest(test_util.ConfigTestCase): doc_value_check = self.assertNotIn real_value_check = self.assertIn - doc_value_check("certbot(-auto)", ua) doc_value_check("OS_NAME OS_VERSION", ua) doc_value_check("major.minor.patchlevel", ua) real_value_check(util.get_os_info_ua(), ua) @@ -58,7 +57,7 @@ class RegisterTest(test_util.ConfigTestCase): """Tests for certbot._internal.client.register.""" def setUp(self): - super(RegisterTest, self).setUp() + super().setUp() self.config.rsa_key_size = 1024 self.config.register_unsafely_without_email = False self.config.email = "alias@example.com" @@ -94,11 +93,11 @@ class RegisterTest(test_util.ConfigTestCase): with mock.patch("certbot._internal.eff.prepare_subscription") as mock_prepare: mock_client().new_account_and_tos.side_effect = errors.Error self.assertRaises(errors.Error, self._call) - self.assertFalse(mock_prepare.called) + self.assertIs(mock_prepare.called, False) mock_client().new_account_and_tos.side_effect = None self._call() - self.assertTrue(mock_prepare.called) + self.assertIs(mock_prepare.called, True) def test_it(self): with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client: @@ -118,7 +117,7 @@ class RegisterTest(test_util.ConfigTestCase): mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self._call() self.assertEqual(mock_get_email.call_count, 1) - self.assertTrue(mock_prepare.called) + self.assertIs(mock_prepare.called, True) def test_email_invalid_noninteractive(self): from acme import messages @@ -145,7 +144,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.dry_run = False self._call() mock_logger.debug.assert_called_once_with(mock.ANY) - self.assertTrue(mock_prepare.called) + self.assertIs(mock_prepare.called, True) @mock.patch("certbot._internal.client.display_ops.get_email") def test_dry_run_no_staging_account(self, mock_get_email): @@ -156,7 +155,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.dry_run = True self._call() # check Certbot did not ask the user to provide an email - self.assertFalse(mock_get_email.called) + self.assertIs(mock_get_email.called, False) # check Certbot created an account with no email. Contact should return empty self.assertFalse(mock_client().new_account_and_tos.call_args[0][0].contact) @@ -173,7 +172,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.eab_hmac_key = "J2OAqW4MHXsrHVa_PVg0Y-L_R4SYw0_aL1le6mfblbE" self._call() - self.assertTrue(mock_eab_from_data.called) + self.assertIs(mock_eab_from_data.called, True) def test_without_eab_arguments(self): with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client: @@ -185,7 +184,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.eab_hmac_key = None self._call() - self.assertFalse(mock_eab_from_data.called) + self.assertIs(mock_eab_from_data.called, False) def test_external_account_required_without_eab_arguments(self): with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client: @@ -210,14 +209,14 @@ class RegisterTest(test_util.ConfigTestCase): with mock.patch("certbot._internal.eff.handle_subscription") as mock_handle: mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(messages.Error, self._call) - self.assertFalse(mock_handle.called) + self.assertIs(mock_handle.called, False) class ClientTestCommon(test_util.ConfigTestCase): """Common base class for certbot._internal.client.Client tests.""" def setUp(self): - super(ClientTestCommon, self).setUp() + super().setUp() self.config.no_verify_ssl = False self.config.allow_subset_of_names = False @@ -236,7 +235,7 @@ class ClientTest(ClientTestCommon): """Tests for certbot._internal.client.Client.""" def setUp(self): - super(ClientTest, self).setUp() + super().setUp() self.config.allow_subset_of_names = False self.config.dry_run = False @@ -247,7 +246,7 @@ class ClientTest(ClientTestCommon): def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] - self.assertTrue(net.verify_ssl) + self.assertIs(net.verify_ssl, True) def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() @@ -582,7 +581,7 @@ class EnhanceConfigTest(ClientTestCommon): """Tests for certbot._internal.client.Client.enhance_config.""" def setUp(self): - super(EnhanceConfigTest, self).setUp() + super().setUp() self.config.hsts = False self.config.redirect = False @@ -609,7 +608,7 @@ class EnhanceConfigTest(ClientTestCommon): def test_already_exists_header(self, mock_log): self.config.hsts = True self._test_with_already_existing() - self.assertTrue(mock_log.warning.called) + self.assertIs(mock_log.warning.called, True) self.assertEqual(mock_log.warning.call_args[0][1], 'Strict-Transport-Security') @@ -617,7 +616,7 @@ class EnhanceConfigTest(ClientTestCommon): def test_already_exists_redirect(self, mock_log): self.config.redirect = True self._test_with_already_existing() - self.assertTrue(mock_log.warning.called) + self.assertIs(mock_log.warning.called, True) self.assertEqual(mock_log.warning.call_args[0][1], 'redirect') @@ -625,13 +624,13 @@ class EnhanceConfigTest(ClientTestCommon): def test_config_set_no_warning_redirect(self, mock_log): self.config.redirect = False self._test_with_already_existing() - self.assertFalse(mock_log.warning.called) + self.assertIs(mock_log.warning.called, False) @mock.patch("certbot._internal.client.logger") def test_no_warn_redirect(self, mock_log): self.config.redirect = None self._test_with_all_supported() - self.assertFalse(mock_log.warning.called) + self.assertIs(mock_log.warning.called, False) def test_no_ask_hsts(self): self.config.hsts = True @@ -684,7 +683,7 @@ class EnhanceConfigTest(ClientTestCommon): def _test_error_with_rollback(self): self._test_error() - self.assertTrue(self.client.installer.restart.called) + self.assertIs(self.client.installer.restart.called, True) def _test_error(self): self.config.redirect = True diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index ea48c9d1c..9aab49c34 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -34,7 +34,7 @@ ADMINS_SID = 'S-1-5-32-544' class WindowsChmodTests(TempDirTestCase): """Unit tests for Windows chmod function in filesystem module""" def setUp(self): - super(WindowsChmodTests, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_symlink_resolution(self): @@ -157,17 +157,17 @@ class UmaskTest(TempDirTestCase): try: dir1 = os.path.join(self.tempdir, 'probe1') filesystem.mkdir(dir1) - self.assertTrue(filesystem.check_mode(dir1, 0o755)) + self.assertIs(filesystem.check_mode(dir1, 0o755), True) filesystem.umask(0o077) dir2 = os.path.join(self.tempdir, 'dir2') filesystem.mkdir(dir2) - self.assertTrue(filesystem.check_mode(dir2, 0o700)) + self.assertIs(filesystem.check_mode(dir2, 0o700), True) dir3 = os.path.join(self.tempdir, 'dir3') filesystem.mkdir(dir3, mode=0o777) - self.assertTrue(filesystem.check_mode(dir3, 0o700)) + self.assertIs(filesystem.check_mode(dir3, 0o700), True) finally: filesystem.umask(previous_umask) @@ -177,17 +177,17 @@ class UmaskTest(TempDirTestCase): try: file1 = os.path.join(self.tempdir, 'probe1') UmaskTest._create_file(file1) - self.assertTrue(filesystem.check_mode(file1, 0o755)) + self.assertIs(filesystem.check_mode(file1, 0o755), True) filesystem.umask(0o077) file2 = os.path.join(self.tempdir, 'probe2') UmaskTest._create_file(file2) - self.assertTrue(filesystem.check_mode(file2, 0o700)) + self.assertIs(filesystem.check_mode(file2, 0o700), True) file3 = os.path.join(self.tempdir, 'probe3') UmaskTest._create_file(file3) - self.assertTrue(filesystem.check_mode(file3, 0o700)) + self.assertIs(filesystem.check_mode(file3, 0o700), True) finally: filesystem.umask(previous_umask) @@ -203,7 +203,7 @@ class UmaskTest(TempDirTestCase): class ComputePrivateKeyModeTest(TempDirTestCase): def setUp(self): - super(ComputePrivateKeyModeTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_compute_private_key_mode(self): @@ -351,7 +351,7 @@ class MakedirsTests(test_util.TempDirTestCase): class CopyOwnershipAndModeTest(test_util.TempDirTestCase): """Tests about copy_ownership_and_apply_mode, copy_ownership_and_mode and has_same_ownership""" def setUp(self): - super(CopyOwnershipAndModeTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') @@ -400,7 +400,7 @@ class CopyOwnershipAndModeTest(test_util.TempDirTestCase): util.safe_open(path1, 'w').close() util.safe_open(path2, 'w').close() - self.assertTrue(filesystem.has_same_ownership(path1, path2)) + self.assertIs(filesystem.has_same_ownership(path1, path2), True) @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') def test_copy_ownership_and_mode_windows(self): @@ -408,8 +408,8 @@ class CopyOwnershipAndModeTest(test_util.TempDirTestCase): dst = _create_probe(self.tempdir, name='dst') filesystem.chmod(src, 0o700) - self.assertTrue(filesystem.check_mode(src, 0o700)) - self.assertTrue(filesystem.check_mode(dst, 0o744)) + self.assertIs(filesystem.check_mode(src, 0o700), True) + self.assertIs(filesystem.check_mode(dst, 0o744), True) # Checking an actual change of owner is tricky during a unit test, since we do not know # if any user exists beside the current one. So we mock _copy_win_ownership. It's behavior @@ -418,24 +418,24 @@ class CopyOwnershipAndModeTest(test_util.TempDirTestCase): filesystem.copy_ownership_and_mode(src, dst) mock_copy_owner.assert_called_once_with(src, dst) - self.assertTrue(filesystem.check_mode(dst, 0o700)) + self.assertIs(filesystem.check_mode(dst, 0o700), True) class CheckPermissionsTest(test_util.TempDirTestCase): """Tests relative to functions that check modes.""" def setUp(self): - super(CheckPermissionsTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_check_mode(self): - self.assertTrue(filesystem.check_mode(self.probe_path, 0o744)) + self.assertIs(filesystem.check_mode(self.probe_path, 0o744), True) filesystem.chmod(self.probe_path, 0o700) self.assertFalse(filesystem.check_mode(self.probe_path, 0o744)) @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') def test_check_owner_windows(self): - self.assertTrue(filesystem.check_owner(self.probe_path)) + self.assertIs(filesystem.check_owner(self.probe_path), True) system = win32security.ConvertStringSidToSid(SYSTEM_SID) security = win32security.SECURITY_ATTRIBUTES().SECURITY_DESCRIPTOR @@ -447,7 +447,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): @unittest.skipUnless(POSIX_MODE, reason='Test specific to Linux security') def test_check_owner_linux(self): - self.assertTrue(filesystem.check_owner(self.probe_path)) + self.assertIs(filesystem.check_owner(self.probe_path), True) import os as std_os # pylint: disable=os-module-forbidden # See related inline comment in certbot.compat.filesystem.check_owner method @@ -459,7 +459,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): self.assertFalse(filesystem.check_owner(self.probe_path)) def test_check_permissions(self): - self.assertTrue(filesystem.check_permissions(self.probe_path, 0o744)) + self.assertIs(filesystem.check_permissions(self.probe_path, 0o744), True) with mock.patch('certbot.compat.filesystem.check_mode') as mock_mode: mock_mode.return_value = False @@ -471,7 +471,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_check_min_permissions(self): filesystem.chmod(self.probe_path, 0o744) - self.assertTrue(filesystem.has_min_permissions(self.probe_path, 0o744)) + self.assertIs(filesystem.has_min_permissions(self.probe_path, 0o744), True) filesystem.chmod(self.probe_path, 0o700) self.assertFalse(filesystem.has_min_permissions(self.probe_path, 0o744)) @@ -481,7 +481,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_is_world_reachable(self): filesystem.chmod(self.probe_path, 0o744) - self.assertTrue(filesystem.has_world_permissions(self.probe_path)) + self.assertIs(filesystem.has_world_permissions(self.probe_path), True) filesystem.chmod(self.probe_path, 0o700) self.assertFalse(filesystem.has_world_permissions(self.probe_path)) @@ -500,13 +500,13 @@ class OsReplaceTest(test_util.TempDirTestCase): filesystem.replace(src, dst) self.assertFalse(os.path.exists(src)) - self.assertTrue(os.path.exists(dst)) + self.assertIs(os.path.exists(dst), True) class RealpathTest(test_util.TempDirTestCase): """Tests for realpath method""" def setUp(self): - super(RealpathTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_symlink_resolution(self): @@ -542,7 +542,7 @@ class RealpathTest(test_util.TempDirTestCase): with self.assertRaises(RuntimeError) as error: filesystem.realpath(link1_path) - self.assertTrue('link1 is a loop!' in str(error.exception)) + self.assertIn('link1 is a loop!', str(error.exception)) class IsExecutableTest(test_util.TempDirTestCase): @@ -578,7 +578,7 @@ class IsExecutableTest(test_util.TempDirTestCase): with _fix_windows_runtime(): mock_access.return_value = True mock_isfile.return_value = True - self.assertTrue(filesystem.is_executable("/path/to/exe")) + self.assertIs(filesystem.is_executable("/path/to/exe"), True) @mock.patch("certbot.compat.filesystem.os.path.isfile") @mock.patch("certbot.compat.filesystem.os.access") @@ -586,7 +586,7 @@ class IsExecutableTest(test_util.TempDirTestCase): with _fix_windows_runtime(): mock_access.return_value = True mock_isfile.return_value = True - self.assertTrue(filesystem.is_executable("exe")) + self.assertIs(filesystem.is_executable("exe"), True) @mock.patch("certbot.compat.filesystem.os.path.isfile") @mock.patch("certbot.compat.filesystem.os.access") diff --git a/certbot/tests/compat/misc_test.py b/certbot/tests/compat/misc_test.py index 642f395ba..e87498cbe 100644 --- a/certbot/tests/compat/misc_test.py +++ b/certbot/tests/compat/misc_test.py @@ -45,4 +45,4 @@ class ExecuteTest(unittest.TestCase): mock_logger.info.assert_any_call(mock.ANY, mock.ANY, mock.ANY, stdout) if stderr or returncode: - self.assertTrue(mock_logger.error.called) + self.assertIs(mock_logger.error.called, True) diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index 1f8a0e803..e23c50afb 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -17,7 +17,7 @@ class NamespaceConfigTest(test_util.ConfigTestCase): """Tests for certbot._internal.configuration.NamespaceConfig.""" def setUp(self): - super(NamespaceConfigTest, self).setUp() + super().setUp() self.config.foo = 'bar' # pylint: disable=blacklisted-name self.config.server = 'https://acme-server.org:443/new' self.config.https_port = 1234 diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 3b9c973f7..432fdbe53 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -36,7 +36,7 @@ CERT_ALT_ISSUER = test_util.load_vector('cert_intermediate_2.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" def setUp(self): - super(InitSaveKeyTest, self).setUp() + super().setUp() self.workdir = os.path.join(self.tempdir, 'workdir') filesystem.mkdir(self.workdir, mode=0o700) @@ -46,7 +46,7 @@ class InitSaveKeyTest(test_util.TempDirTestCase): mock.Mock(strict_permissions=True), interfaces.IConfig) def tearDown(self): - super(InitSaveKeyTest, self).tearDown() + super().tearDown() logging.disable(logging.NOTSET) @@ -60,7 +60,7 @@ class InitSaveKeyTest(test_util.TempDirTestCase): mock_make.return_value = b'key_pem' key = self._call(1024, self.workdir) self.assertEqual(key.pem, b'key_pem') - self.assertTrue('key-certbot.pem' in key.file) + self.assertIn('key-certbot.pem', key.file) self.assertTrue(os.path.exists(os.path.join(self.workdir, key.file))) @mock.patch('certbot.crypto_util.make_key') @@ -73,7 +73,7 @@ class InitSaveCSRTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_csr.""" def setUp(self): - super(InitSaveCSRTest, self).setUp() + super().setUp() zope.component.provideUtility( mock.Mock(strict_permissions=True), interfaces.IConfig) @@ -89,7 +89,7 @@ class InitSaveCSRTest(test_util.TempDirTestCase): mock.Mock(pem='dummy_key'), 'example.com', self.tempdir) self.assertEqual(csr.data, b'csr_pem') - self.assertTrue('csr-certbot.pem' in csr.file) + self.assertIn('csr-certbot.pem', csr.file) class ValidCSRTest(unittest.TestCase): @@ -251,7 +251,7 @@ class VerifyRenewableCertTest(VerifyCertSetup): return verify_renewable_cert(renewable_cert) def test_verify_renewable_cert(self): - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) @mock.patch('certbot.crypto_util.verify_renewable_cert_sig', side_effect=errors.Error("")) def test_verify_renewable_cert_failure(self, unused_verify_renewable_cert_sign): @@ -266,14 +266,14 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): return verify_renewable_cert_sig(renewable_cert) def test_cert_sig_match(self): - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) def test_cert_sig_match_ec(self): renewable_cert = mock.MagicMock() renewable_cert.cert_path = P256_CERT_PATH renewable_cert.chain_path = P256_CERT_PATH renewable_cert.key_path = P256_KEY - self.assertEqual(None, self._call(renewable_cert)) + self.assertIsNone(self._call(renewable_cert)) def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert_path = test_util.vector_path('cert_512_bad.pem') @@ -288,7 +288,7 @@ class VerifyFullchainTest(VerifyCertSetup): return verify_fullchain(renewable_cert) def test_fullchain_matches(self): - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) def test_fullchain_mismatch(self): self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) @@ -308,7 +308,7 @@ class VerifyCertMatchesPrivKeyTest(VerifyCertSetup): def test_cert_priv_key_match(self): self.renewable_cert.cert = SS_CERT_PATH self.renewable_cert.privkey = RSA2048_KEY_PATH - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) def test_cert_priv_key_mismatch(self): self.bad_renewable_cert.privkey = RSA256_KEY_PATH diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 8ae6290bf..75b11d1d7 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -1,4 +1,6 @@ """Test certbot._internal.display.completer.""" +from typing import List + try: import readline # pylint: disable=import-error except ImportError: @@ -23,14 +25,14 @@ class CompleterTest(test_util.TempDirTestCase): """Test certbot._internal.display.completer.Completer.""" def setUp(self): - super(CompleterTest, self).setUp() + super().setUp() # directories must end with os.sep for completer to # search inside the directory for possible completions if self.tempdir[-1] != os.sep: self.tempdir += os.sep - self.paths = [] # type: List[str] + self.paths: List[str] = [] # create some files and directories in temp_dir for c in string.ascii_lowercase: path = os.path.join(self.tempdir, c) @@ -48,12 +50,12 @@ class CompleterTest(test_util.TempDirTestCase): for i in range(num_paths): completion = my_completer.complete(self.tempdir, i) - self.assertTrue(completion in self.paths) + self.assertIn(completion, self.paths) self.paths.remove(completion) - self.assertFalse(self.paths) + self.assertEqual(len(self.paths), 0) completion = my_completer.complete(self.tempdir, num_paths) - self.assertEqual(completion, None) + self.assertIsNone(completion) @unittest.skipIf('readline' not in sys.modules, reason='Not relevant if readline is not available.') @@ -96,7 +98,7 @@ class CompleterTest(test_util.TempDirTestCase): with completer.Completer(): pass - self.assertTrue(mock_readline.parse_and_bind.called) + self.assertIs(mock_readline.parse_and_bind.called, True) def enable_tab_completion(unused_command): diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index ce60a5f0b..aeb3ea525 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -61,9 +61,9 @@ class GetEmailTest(unittest.TestCase): with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.return_value = True self._call() - self.assertTrue(invalid_txt not in mock_input.call_args[0][0]) + self.assertNotIn(invalid_txt, mock_input.call_args[0][0]) self._call(invalid=True) - self.assertTrue(invalid_txt in mock_input.call_args[0][0]) + self.assertIn(invalid_txt, mock_input.call_args[0][0]) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_optional_flag(self, mock_get_utility): @@ -73,8 +73,7 @@ class GetEmailTest(unittest.TestCase): mock_safe_email.side_effect = [False, True] self._call(optional=False) for call in mock_input.call_args_list: - self.assertTrue( - "--register-unsafely-without-email" not in call[0][0]) + self.assertNotIn("--register-unsafely-without-email", call[0][0]) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_optional_invalid_unsafe(self, mock_get_utility): @@ -84,13 +83,13 @@ class GetEmailTest(unittest.TestCase): with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.side_effect = [False, True] self._call(invalid=True) - self.assertTrue(invalid_txt in mock_input.call_args[0][0]) + self.assertIn(invalid_txt, mock_input.call_args[0][0]) class ChooseAccountTest(test_util.TempDirTestCase): """Tests for certbot.display.ops.choose_account.""" def setUp(self): - super(ChooseAccountTest, self).setUp() + super().setUp() zope.component.provideUtility(display_util.FileDisplay(sys.stdout, False)) @@ -128,7 +127,7 @@ class ChooseAccountTest(test_util.TempDirTestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_cancel(self, mock_util): mock_util().menu.return_value = (display_util.CANCEL, 1) - self.assertTrue(self._call([self.acc1, self.acc2]) is None) + self.assertIsNone(self._call([self.acc1, self.acc2])) class GenHttpsNamesTest(unittest.TestCase): @@ -210,8 +209,7 @@ class ChooseNamesTest(unittest.TestCase): actual_doms = self._call(self.mock_install) self.assertEqual(mock_util().input.call_count, 1) self.assertEqual(actual_doms, [domain]) - self.assertTrue( - "configuration files" in mock_util().input.call_args[0][0]) + self.assertIn("configuration files", mock_util().input.call_args[0][0]) def test_sort_names_trivial(self): from certbot.display.ops import _sort_names @@ -353,7 +351,7 @@ class SuccessInstallationTest(unittest.TestCase): arg = mock_util().notification.call_args_list[0][0][0] for name in names: - self.assertTrue(name in arg) + self.assertIn(name, arg) class SuccessRenewalTest(unittest.TestCase): @@ -374,7 +372,7 @@ class SuccessRenewalTest(unittest.TestCase): arg = mock_util().notification.call_args_list[0][0][0] for name in names: - self.assertTrue(name in arg) + self.assertIn(name, arg) class SuccessRevocationTest(unittest.TestCase): """Test the success revocation message.""" @@ -478,8 +476,8 @@ class ChooseValuesTest(unittest.TestCase): mock_util().checklist.return_value = (display_util.OK, [items[2]]) result = self._call(items, None) self.assertEqual(result, [items[2]]) - self.assertTrue(mock_util().checklist.called) - self.assertEqual(mock_util().checklist.call_args[0][0], None) + self.assertIs(mock_util().checklist.called, True) + self.assertIsNone(mock_util().checklist.call_args[0][0]) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_choose_names_success_question(self, mock_util): @@ -488,7 +486,7 @@ class ChooseValuesTest(unittest.TestCase): mock_util().checklist.return_value = (display_util.OK, [items[1]]) result = self._call(items, question) self.assertEqual(result, [items[1]]) - self.assertTrue(mock_util().checklist.called) + self.assertIs(mock_util().checklist.called, True) self.assertEqual(mock_util().checklist.call_args[0][0], question) @test_util.patch_get_utility("certbot.display.ops.z_util") @@ -498,7 +496,7 @@ class ChooseValuesTest(unittest.TestCase): mock_util().checklist.return_value = (display_util.CANCEL, []) result = self._call(items, question) self.assertEqual(result, []) - self.assertTrue(mock_util().checklist.called) + self.assertIs(mock_util().checklist.called, True) self.assertEqual(mock_util().checklist.call_args[0][0], question) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 30b33bbc9..ca7ecf908 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -65,7 +65,7 @@ class FileOutputDisplayTest(unittest.TestCase): """ def setUp(self): - super(FileOutputDisplayTest, self).setUp() + super().setUp() self.mock_stdout = mock.MagicMock() self.displayer = display_util.FileDisplay(self.mock_stdout, False) @@ -74,7 +74,7 @@ class FileOutputDisplayTest(unittest.TestCase): self.displayer.notification("message", False) string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) mock_logger.debug.assert_called_with("Notifying user: %s", "message") def test_notification_pause(self): @@ -82,25 +82,25 @@ class FileOutputDisplayTest(unittest.TestCase): with mock.patch(input_with_timeout, return_value="enter"): self.displayer.notification("message", force_interactive=True) - self.assertTrue("message" in self.mock_stdout.write.call_args[0][0]) + self.assertIn("message", self.mock_stdout.write.call_args[0][0]) def test_notification_noninteractive(self): self._force_noninteractive(self.displayer.notification, "message") string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) def test_notification_noninteractive2(self): # The main purpose of this test is to make sure we only call # logger.warning once which _force_noninteractive checks internally self._force_noninteractive(self.displayer.notification, "message") string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) self.assertTrue(self.displayer.skipped_interaction) self._force_noninteractive(self.displayer.notification, "message2") string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message2" in string) + self.assertIn("message2", string) def test_notification_decoration(self): from certbot.compat import os @@ -110,7 +110,8 @@ class FileOutputDisplayTest(unittest.TestCase): self.displayer.notification("message2", pause=False) string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("- - - " in string and ("message2" + os.linesep) in string) + self.assertIn("- - - ", string) + self.assertIn("message2" + os.linesep, string) @mock.patch("certbot.display.util." "FileDisplay._get_valid_int_ans") @@ -265,7 +266,7 @@ class FileOutputDisplayTest(unittest.TestCase): result = func(*args, **kwargs) if skipped_interaction: - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) else: self.assertEqual(mock_logger.warning.call_count, 1) @@ -331,7 +332,7 @@ class FileOutputDisplayTest(unittest.TestCase): # force_interactive to prevent workflow regressions. for name in interfaces.IDisplay.names(): arg_spec = inspect.getfullargspec(getattr(self.displayer, name)) - self.assertTrue("force_interactive" in arg_spec.args) + self.assertIn("force_interactive", arg_spec.args) class NoninteractiveDisplayTest(unittest.TestCase): @@ -345,7 +346,7 @@ class NoninteractiveDisplayTest(unittest.TestCase): self.displayer.notification("message", 10) string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) mock_logger.debug.assert_called_with("Notifying user: %s", "message") def test_notification_decoration(self): @@ -401,7 +402,7 @@ class NoninteractiveDisplayTest(unittest.TestCase): method = getattr(self.displayer, name) # asserts method accepts arbitrary keyword arguments result = inspect.getfullargspec(method).varkw - self.assertFalse(result is None) + self.assertIsNotNone(result) class SeparateListInputTest(unittest.TestCase): diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index dc6b6b944..0527d87d9 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -22,7 +22,7 @@ _KEY = josepy.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) class SubscriptionTest(test_util.ConfigTestCase): """Abstract class for subscription tests.""" def setUp(self): - super(SubscriptionTest, self).setUp() + super().setUp() self.account = account.Account( regr=messages.RegistrationResource( uri=None, body=messages.Registration(), @@ -50,7 +50,7 @@ class PrepareSubscriptionTest(SubscriptionTest): self._call() actual = mock_notify.call_args[0][0] expected_part = "because you didn't provide an e-mail address" - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) self.assertIsNone(self.account.meta.register_to_eff) @test_util.patch_get_utility() @@ -92,7 +92,7 @@ class PrepareSubscriptionTest(SubscriptionTest): call_args, call_kwargs = mock_get_utility().yesno.call_args actual = call_args[0] expected_part = 'Electronic Frontier Foundation' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) self.assertFalse(call_kwargs.get('default', True)) @@ -105,7 +105,7 @@ class HandleSubscriptionTest(SubscriptionTest): @mock.patch('certbot._internal.eff.subscribe') def test_no_subscribe(self, mock_subscribe): self._call() - self.assertFalse(mock_subscribe.called) + self.assertIs(mock_subscribe.called, False) @mock.patch('certbot._internal.eff.subscribe') def test_subscribe(self, mock_subscribe): @@ -140,7 +140,7 @@ class SubscribeTest(unittest.TestCase): self.assertEqual(call_args[0], constants.EFF_SUBSCRIBE_URI) data = call_kwargs.get('data') - self.assertFalse(data is None) + self.assertIsNotNone(data) self.assertEqual(data.get('email'), self.email) def test_bad_status(self): @@ -148,7 +148,7 @@ class SubscribeTest(unittest.TestCase): self._call() actual = self._get_reported_message() expected_part = 'because your e-mail address appears to be invalid.' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) def test_not_ok(self): self.response.ok = False @@ -156,21 +156,21 @@ class SubscribeTest(unittest.TestCase): self._call() actual = self._get_reported_message() unexpected_part = 'because' - self.assertFalse(unexpected_part in actual) + self.assertNotIn(unexpected_part, actual) def test_response_not_json(self): self.response.json.side_effect = ValueError() self._call() actual = self._get_reported_message() expected_part = 'problem' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) def test_response_json_missing_status_element(self): self.json.clear() self._call() actual = self._get_reported_message() expected_part = 'problem' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) def _get_reported_message(self): self.assertTrue(self.mock_notify.called) @@ -179,7 +179,7 @@ class SubscribeTest(unittest.TestCase): @test_util.patch_get_utility() def test_subscribe(self, mock_get_utility): self._call() - self.assertFalse(mock_get_utility.called) + self.assertIs(mock_get_utility.called, False) if __name__ == '__main__': diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 9ccd63a3a..ee4c2215d 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -3,6 +3,7 @@ import contextlib import signal import sys import unittest +from typing import Callable, Dict, Union try: import mock @@ -27,7 +28,7 @@ def set_signals(sig_handler_dict): def signal_receiver(signums): """Context manager to catch signals""" signals = [] - prev_handlers = get_signals(signums) # type: Dict[int, Union[int, None, Callable]] + prev_handlers: Dict[int, Union[int, None, Callable]] = get_signals(signums) set_signals({s: lambda s, _: signals.append(s) for s in signums}) yield signals set_signals(prev_handlers) @@ -119,7 +120,7 @@ class ErrorHandlerTest(unittest.TestCase): sys.exit(0) except SystemExit: pass - self.assertFalse(self.init_func.called) + self.assertIs(self.init_func.called, False) def test_regular_exit(self): func = mock.MagicMock() @@ -135,7 +136,7 @@ class ExitHandlerTest(ErrorHandlerTest): def setUp(self): from certbot._internal import error_handler - super(ExitHandlerTest, self).setUp() + super().setUp() self.handler = error_handler.ExitHandler(self.init_func, *self.init_args, **self.init_kwargs) diff --git a/certbot/tests/helpful_test.py b/certbot/tests/helpful_test.py index 1a5c2bea6..0abe277bf 100644 --- a/certbot/tests/helpful_test.py +++ b/certbot/tests/helpful_test.py @@ -18,10 +18,10 @@ class TestScanningFlags(unittest.TestCase): arg_parser = HelpfulArgumentParser(['run'], {}) detected_flag = arg_parser.prescan_for_flag('--help', ['all', 'certonly']) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) detected_flag = arg_parser.prescan_for_flag('-h', ['all, certonly']) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) def test_prescan_unvalid_topic(self): arg_parser = HelpfulArgumentParser(['--help', 'all'], {}) @@ -30,7 +30,7 @@ class TestScanningFlags(unittest.TestCase): self.assertIs(detected_flag, True) detected_flag = arg_parser.prescan_for_flag('-h', arg_parser.help_topics) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) def test_prescan_valid_topic(self): arg_parser = HelpfulArgumentParser(['-h', 'all'], {}) @@ -39,7 +39,7 @@ class TestScanningFlags(unittest.TestCase): self.assertEqual(detected_flag, 'all') detected_flag = arg_parser.prescan_for_flag('--help', arg_parser.help_topics) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) class TestDetermineVerbs(unittest.TestCase): '''Tests for determine_verb methods of HelpfulArgumentParser''' @@ -90,7 +90,7 @@ class TestAdd(unittest.TestCase): metavar="EAB_KID", help="Key Identifier for External Account Binding") parsed_args = arg_parser.parser.parse_args(["--eab-kid", None]) - self.assertIs(parsed_args.eab_kid, None) + self.assertIsNone(parsed_args.eab_kid) self.assertTrue(hasattr(parsed_args, 'eab_kid')) @@ -115,7 +115,7 @@ class TestAddGroup(unittest.TestCase): self.assertTrue(arg_parser.groups["run"]) arg_parser.add_group("certonly", description="description of certonly") with self.assertRaises(KeyError): - self.assertFalse(arg_parser.groups["certonly"]) + self.assertIs(arg_parser.groups["certonly"], False) class TestParseArgsErrors(unittest.TestCase): diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 06a641216..bcef2e398 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -59,7 +59,7 @@ class ValidateHookTest(test_util.TempDirTestCase): @mock.patch("certbot._internal.hooks._prog") def test_unset(self, mock_prog): self._call(None, "foo") - self.assertFalse(mock_prog.called) + self.assertIs(mock_prog.called, False) class HookTest(test_util.ConfigTestCase): @@ -93,7 +93,7 @@ class PreHookTest(HookTest): return pre_hook(*args, **kwargs) def setUp(self): - super(PreHookTest, self).setUp() + super().setUp() self.config.pre_hook = "foo" filesystem.makedirs(self.config.renewal_pre_hooks_dir) @@ -106,7 +106,7 @@ class PreHookTest(HookTest): def tearDown(self): # Reset this value so it's unmodified for future tests self._reset_pre_hook_already() - super(PreHookTest, self).tearDown() + super().tearDown() def _reset_pre_hook_already(self): from certbot._internal.hooks import executed_pre_hooks @@ -132,8 +132,8 @@ class PreHookTest(HookTest): with mock.patch("certbot._internal.hooks.logger") as mock_logger: mock_execute = self._call_with_mock_execute(self.config) - self.assertFalse(mock_execute.called) - self.assertFalse(mock_logger.info.called) + self.assertIs(mock_execute.called, False) + self.assertIs(mock_logger.info.called, False) def test_renew_disabled_dir_hooks(self): self.config.directory_hooks = False @@ -158,7 +158,7 @@ class PreHookTest(HookTest): def _test_no_executions_common(self): with mock.patch("certbot._internal.hooks.logger") as mock_logger: mock_execute = self._call_with_mock_execute(self.config) - self.assertFalse(mock_execute.called) + self.assertIs(mock_execute.called, False) self.assertTrue(mock_logger.info.called) @@ -171,7 +171,7 @@ class PostHookTest(HookTest): return post_hook(*args, **kwargs) def setUp(self): - super(PostHookTest, self).setUp() + super().setUp() self.config.post_hook = "bar" filesystem.makedirs(self.config.renewal_post_hooks_dir) @@ -184,7 +184,7 @@ class PostHookTest(HookTest): def tearDown(self): # Reset this value so it's unmodified for future tests self._reset_post_hook_eventually() - super(PostHookTest, self).tearDown() + super().tearDown() def _reset_post_hook_eventually(self): from certbot._internal.hooks import post_hooks @@ -266,8 +266,8 @@ class RunSavedPostHooksTest(HookTest): return self._call_with_mock_execute(*args, **kwargs) def setUp(self): - super(RunSavedPostHooksTest, self).setUp() - self.eventually = [] # type: List[str] + super().setUp() + self.eventually: List[str] = [] def test_empty(self): self.assertFalse(self._call_with_mock_execute_and_eventually().called) @@ -319,7 +319,7 @@ class RenewalHookTest(HookTest): return mock_execute def setUp(self): - super(RenewalHookTest, self).setUp() + super().setUp() self.vars_to_clear = set( var for var in ("RENEWED_DOMAINS", "RENEWED_LINEAGE",) if var not in os.environ) @@ -327,7 +327,7 @@ class RenewalHookTest(HookTest): def tearDown(self): for var in self.vars_to_clear: os.environ.pop(var, None) - super(RenewalHookTest, self).tearDown() + super().tearDown() class DeployHookTest(RenewalHookTest): @@ -344,7 +344,7 @@ class DeployHookTest(RenewalHookTest): self.config.dry_run = True mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) + self.assertIs(mock_execute.called, False) self.assertTrue(mock_logger.warning.called) @mock.patch("certbot._internal.hooks.logger") @@ -352,8 +352,8 @@ class DeployHookTest(RenewalHookTest): self.config.deploy_hook = None mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) - self.assertFalse(mock_logger.info.called) + self.assertIs(mock_execute.called, False) + self.assertIs(mock_logger.info.called, False) def test_success(self): domains = ["example.org", "example.net"] @@ -373,7 +373,7 @@ class RenewHookTest(RenewalHookTest): return renew_hook(*args, **kwargs) def setUp(self): - super(RenewHookTest, self).setUp() + super().setUp() self.config.renew_hook = "foo" filesystem.makedirs(self.config.renewal_deploy_hooks_dir) @@ -392,7 +392,7 @@ class RenewHookTest(RenewalHookTest): self.config.dry_run = True mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) + self.assertIs(mock_execute.called, False) self.assertEqual(mock_logger.warning.call_count, 2) def test_no_hooks(self): @@ -402,8 +402,8 @@ class RenewHookTest(RenewalHookTest): with mock.patch("certbot._internal.hooks.logger") as mock_logger: mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) - self.assertFalse(mock_logger.info.called) + self.assertIs(mock_execute.called, False) + self.assertIs(mock_logger.info.called, False) def test_overlap(self): self.config.renew_hook = self.dir_hook diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 2f887d33e..b45eb8f7a 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -44,7 +44,7 @@ class LockFileTest(test_util.TempDirTestCase): return LockFile(*args, **kwargs) def setUp(self): - super(LockFileTest, self).setUp() + super().setUp() self.lock_path = os.path.join(self.tempdir, 'test.lock') def test_acquire_without_deletion(self): @@ -69,7 +69,7 @@ class LockFileTest(test_util.TempDirTestCase): try: locked_repr = repr(lock_file) self._test_repr_common(lock_file, locked_repr) - self.assertTrue('acquired' in locked_repr) + self.assertIn('acquired', locked_repr) finally: lock_file.release() @@ -78,11 +78,11 @@ class LockFileTest(test_util.TempDirTestCase): lock_file.release() released_repr = repr(lock_file) self._test_repr_common(lock_file, released_repr) - self.assertTrue('released' in released_repr) + self.assertIn('released', released_repr) def _test_repr_common(self, lock_file, lock_repr): - self.assertTrue(lock_file.__class__.__name__ in lock_repr) - self.assertTrue(self.lock_path in lock_repr) + self.assertIn(lock_file.__class__.__name__, lock_repr) + self.assertIn(self.lock_path, lock_repr) @test_util.skip_on_windows( 'Race conditions on lock are specific to the non-blocking file access approach on Linux.') @@ -102,7 +102,7 @@ class LockFileTest(test_util.TempDirTestCase): with mock.patch('certbot._internal.lock.filesystem.os.stat') as mock_stat: mock_stat.side_effect = delete_and_stat self._call(self.lock_path) - self.assertFalse(should_delete) + self.assertEqual(len(should_delete), 0) def test_removed(self): lock_file = self._call(self.lock_path) @@ -120,7 +120,7 @@ class LockFileTest(test_util.TempDirTestCase): try: self._call(self.lock_path) except IOError as err: - self.assertTrue(msg in str(err)) + self.assertIn(msg, str(err)) else: # pragma: no cover self.fail('IOError not raised') @@ -136,7 +136,7 @@ class LockFileTest(test_util.TempDirTestCase): try: self._call(self.lock_path) except OSError as err: - self.assertTrue(msg in str(err)) + self.assertIn(msg, str(err)) else: # pragma: no cover self.fail('OSError not raised') diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 034597a3c..eaf023893 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -5,7 +5,7 @@ import logging.handlers import sys import time import unittest - +from typing import Optional from acme import messages from certbot import errors @@ -28,7 +28,8 @@ class PreArgParseSetupTest(unittest.TestCase): @classmethod def _call(cls, *args, **kwargs): # pylint: disable=unused-argument from certbot._internal.log import pre_arg_parse_setup - return pre_arg_parse_setup() + with mock.patch('builtins.open', mock.mock_open()): + return pre_arg_parse_setup() @mock.patch('certbot._internal.log.sys') @mock.patch('certbot._internal.log.pre_arg_parse_except_hook') @@ -43,16 +44,15 @@ class PreArgParseSetupTest(unittest.TestCase): mock_root_logger.setLevel.assert_called_once_with(logging.DEBUG) self.assertEqual(mock_root_logger.addHandler.call_count, 2) - memory_handler = None # type: Optional[logging.handlers.MemoryHandler] + memory_handler: Optional[logging.handlers.MemoryHandler] = None for call in mock_root_logger.addHandler.call_args_list: handler = call[0][0] if memory_handler is None and isinstance(handler, logging.handlers.MemoryHandler): memory_handler = handler target = memory_handler.target # type: ignore else: - self.assertTrue(isinstance(handler, logging.StreamHandler)) - self.assertTrue( - isinstance(target, logging.StreamHandler)) + self.assertIsInstance(handler, logging.StreamHandler) + self.assertIsInstance(target, logging.StreamHandler) mock_register.assert_called_once_with(logging.shutdown) mock_sys.excepthook(1, 2, 3) @@ -69,7 +69,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): return post_arg_parse_setup(*args, **kwargs) def setUp(self): - super(PostArgParseSetupTest, self).setUp() + super().setUp() self.config.debug = False self.config.max_log_backups = 1000 self.config.quiet = False @@ -90,7 +90,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): self.stream_handler.close() self.temp_handler.close() self.devnull.close() - super(PostArgParseSetupTest, self).tearDown() + super().tearDown() def test_common(self): with mock.patch('certbot._internal.log.logging.getLogger') as mock_get_logger: @@ -136,7 +136,7 @@ class SetupLogFileHandlerTest(test_util.ConfigTestCase): return setup_log_file_handler(*args, **kwargs) def setUp(self): - super(SetupLogFileHandlerTest, self).setUp() + super().setUp() self.config.max_log_backups = 42 @mock.patch('certbot._internal.main.logging.handlers.RotatingFileHandler') @@ -146,7 +146,7 @@ class SetupLogFileHandlerTest(test_util.ConfigTestCase): try: self._call(self.config, 'test.log', '%(message)s') except errors.Error as err: - self.assertTrue('--logs-dir' in str(err)) + self.assertIn('--logs-dir', str(err)) else: # pragma: no cover self.fail('Error not raised.') @@ -343,7 +343,7 @@ class PostArgParseExceptHookTest(unittest.TestCase): mock_logger, output = self._test_common(get_acme_error, debug=False) self._assert_exception_logged(mock_logger.debug, messages.Error) self._assert_quiet_output(mock_logger, output) - self.assertFalse(messages.ERROR_PREFIX in output) + self.assertNotIn(messages.ERROR_PREFIX, output) def test_other_error(self): exc_type = ValueError @@ -385,7 +385,7 @@ class PostArgParseExceptHookTest(unittest.TestCase): def _assert_exception_logged(self, log_func, exc_type): self.assertTrue(log_func.called) call_kwargs = log_func.call_args[1] - self.assertTrue('exc_info' in call_kwargs) + self.assertIn('exc_info', call_kwargs) actual_exc_info = call_kwargs['exc_info'] expected_exc_info = (exc_type, mock.ANY, mock.ANY) @@ -396,9 +396,9 @@ class PostArgParseExceptHookTest(unittest.TestCase): self.assertIn(self.log_path, output) def _assert_quiet_output(self, mock_logger, output): - self.assertFalse(mock_logger.exception.called) + self.assertIs(mock_logger.exception.called, False) self.assertTrue(mock_logger.debug.called) - self.assertTrue(self.error_msg in output) + self.assertIn(self.error_msg, output) class ExitWithAdviceTest(test_util.TempDirTestCase): @@ -413,13 +413,13 @@ class ExitWithAdviceTest(test_util.TempDirTestCase): open(log_file, 'w').close() err_str = self._test_common(log_file) - self.assertTrue('logfiles' not in err_str) - self.assertTrue(log_file in err_str) + self.assertNotIn('logfiles', err_str) + self.assertIn(log_file, err_str) def test_log_dir(self): err_str = self._test_common(self.tempdir) - self.assertTrue('logfiles' in err_str) - self.assertTrue(self.tempdir in err_str) + self.assertIn('logfiles', err_str) + self.assertIn(self.tempdir, err_str) # pylint: disable=inconsistent-return-statements def _test_common(self, *args, **kwargs): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index ddd911c8d..5689c0281 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,8 +1,6 @@ # coding=utf-8 """Tests for certbot._internal.main.""" # pylint: disable=too-many-lines -from __future__ import print_function - import datetime from importlib import reload as reload_module import io @@ -13,6 +11,7 @@ import sys import tempfile import traceback import unittest +from typing import List import josepy as jose import pytz @@ -85,24 +84,24 @@ class TestHandleCerts(unittest.TestCase): mock_set.return_value = False with self.assertRaises(errors.Error) as raised: main._handle_unexpected_key_type_migration(config, cert) - self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + self.assertIn("Please provide both --cert-name and --key-type", str(raised.exception)) mock_set.side_effect = lambda var: var != "certname" with self.assertRaises(errors.Error) as raised: main._handle_unexpected_key_type_migration(config, cert) - self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + self.assertIn("Please provide both --cert-name and --key-type", str(raised.exception)) mock_set.side_effect = lambda var: var != "key_type" with self.assertRaises(errors.Error) as raised: main._handle_unexpected_key_type_migration(config, cert) - self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + self.assertIn("Please provide both --cert-name and --key-type", str(raised.exception)) class RunTest(test_util.ConfigTestCase): """Tests for certbot._internal.main.run.""" def setUp(self): - super(RunTest, self).setUp() + super().setUp() self.domain = 'example.org' patches = [ mock.patch('certbot._internal.main._get_and_save_cert'), @@ -197,7 +196,7 @@ class CertonlyTest(unittest.TestCase): self._call('certonly --webroot -d example.com'.split()) def _assert_no_pause(self, message, pause=True): # pylint: disable=unused-argument - self.assertFalse(pause) + self.assertIs(pause, False) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @mock.patch('certbot._internal.cert_manager.domains_for_certname') @@ -261,8 +260,7 @@ class FindDomainsOrCertnameTest(unittest.TestCase): mock_config = mock.Mock(domains=None, certname=None) mock_choose_names.return_value = "domainname" # pylint: disable=protected-access - self.assertEqual(main._find_domains_or_certname(mock_config, None), - ("domainname", None)) + self.assertEqual(main._find_domains_or_certname(mock_config, None), ("domainname", None)) @mock.patch('certbot.display.ops.choose_names') def test_no_results(self, mock_choose_names): @@ -276,21 +274,20 @@ class FindDomainsOrCertnameTest(unittest.TestCase): mock_config = mock.Mock(domains=None, certname="one.com") mock_domains.return_value = ["one.com", "two.com"] # pylint: disable=protected-access - self.assertEqual(main._find_domains_or_certname(mock_config, None), - (["one.com", "two.com"], "one.com")) + self.assertEqual( + main._find_domains_or_certname(mock_config, None), + (["one.com", "two.com"], "one.com") + ) class RevokeTest(test_util.TempDirTestCase): """Tests for certbot._internal.main.revoke.""" def setUp(self): - super(RevokeTest, self).setUp() + super().setUp() shutil.copy(CERT_PATH, self.tempdir) - self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, - 'cert_512.pem')) - with open(self.tmp_cert_path, 'r') as f: - self.tmp_cert = (self.tmp_cert_path, f.read()) + self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, 'cert_512.pem')) patches = [ mock.patch('acme.client.BackwardsCompatibleClientV2'), @@ -320,6 +317,7 @@ class RevokeTest(test_util.TempDirTestCase): if not args: args = 'revoke --cert-path={0} ' args = args.format(self.tmp_cert_path).split() + cli.set_by_cli.detector = None # required to reset set_by_cli state plugins = disco.PluginsRegistry.find_all() config = configuration.NamespaceConfig( cli.prepare_and_parse_args(plugins, args)) @@ -345,13 +343,44 @@ class RevokeTest(test_util.TempDirTestCase): self.assertEqual(expected, mock_revoke.call_args_list) @mock.patch('certbot._internal.main._delete_if_appropriate') - @mock.patch('certbot._internal.storage.cert_path_for_cert_name') - def test_revoke_by_certname(self, mock_cert_path_for_cert_name, - mock_delete_if_appropriate): + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, + server="https://acme.example") args = 'revoke --cert-name=example.com'.split() - mock_cert_path_for_cert_name.return_value = self.tmp_cert mock_delete_if_appropriate.return_value = False self._call(args) + self.mock_acme_client.assert_called_once_with(mock.ANY, mock.ANY, 'https://acme.example') + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + + @mock.patch('certbot._internal.main._delete_if_appropriate') + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname_and_server(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + """Revoking with --server should use the server from the CLI""" + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, + server="https://acme.example") + args = 'revoke --cert-name=example.com --server https://other.example'.split() + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_acme_client.assert_called_once_with(mock.ANY, mock.ANY, 'https://other.example') + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + + @mock.patch('certbot._internal.main._delete_if_appropriate') + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname_empty_server(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + """Revoking with --cert-name where the lineage server is empty shouldn't crash """ + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, server=None) + args = 'revoke --cert-name=example.com'.split() + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_acme_client.assert_called_once_with( + mock.ANY, mock.ANY, constants.CLI_DEFAULTS['server']) self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) @mock.patch('certbot._internal.main._delete_if_appropriate') @@ -374,7 +403,7 @@ class RevokeTest(test_util.TempDirTestCase): mock_get_utility().yesno.return_value = False mock_delete_if_appropriate.return_value = False self._call() - self.assertFalse(mock_delete.called) + self.assertIs(mock_delete.called, False) class DeleteIfAppropriateTest(test_util.ConfigTestCase): """Tests for certbot._internal.main._delete_if_appropriate """ @@ -484,7 +513,7 @@ class DetermineAccountTest(test_util.ConfigTestCase): """Tests for certbot._internal.main._determine_account.""" def setUp(self): - super(DetermineAccountTest, self).setUp() + super().setUp() self.config.account = None self.config.email = None self.config.register_unsafely_without_email = False @@ -508,13 +537,13 @@ class DetermineAccountTest(test_util.ConfigTestCase): self.config.account = self.accs[1].id self.assertEqual((self.accs[1], None), self._call()) self.assertEqual(self.accs[1].id, self.config.account) - self.assertTrue(self.config.email is None) + self.assertIsNone(self.config.email) def test_single_account(self): self.account_storage.save(self.accs[0], self.mock_client) self.assertEqual((self.accs[0], None), self._call()) self.assertEqual(self.accs[0].id, self.config.account) - self.assertTrue(self.config.email is None) + self.assertIsNone(self.config.email) @mock.patch('certbot._internal.client.display_ops.choose_account') def test_multiple_accounts(self, mock_choose_accounts): @@ -525,7 +554,7 @@ class DetermineAccountTest(test_util.ConfigTestCase): self.assertEqual( set(mock_choose_accounts.call_args[0][0]), set(self.accs)) self.assertEqual(self.accs[1].id, self.config.account) - self.assertTrue(self.config.email is None) + self.assertIsNone(self.config.email) @mock.patch('certbot._internal.client.display_ops.get_email') @mock.patch('certbot._internal.main.display_util.notify') @@ -556,7 +585,7 @@ class MainTest(test_util.ConfigTestCase): """Tests for different commands.""" def setUp(self): - super(MainTest, self).setUp() + super().setUp() filesystem.mkdir(self.config.logs_dir) self.standard_args = ['--config-dir', self.config.config_dir, @@ -569,7 +598,7 @@ class MainTest(test_util.ConfigTestCase): # Reset globals in cli reload_module(cli) - super(MainTest, self).tearDown() + super().tearDown() def _call(self, args, stdout=None, mockisfile=False): """Run the cli with output streams, actual client and optionally @@ -623,7 +652,7 @@ class MainTest(test_util.ConfigTestCase): pass finally: output = toy_out.getvalue() or toy_err.getvalue() - self.assertTrue("certbot" in output, "Output is {0}".format(output)) + self.assertIn("certbot", output, "Output is {0}".format(output)) def _cli_missing_flag(self, args, message): "Ensure that a particular error raises a missing cli flag error containing message" @@ -633,8 +662,8 @@ class MainTest(test_util.ConfigTestCase): main.main(self.standard_args + args[:]) # NOTE: parser can alter its args! except errors.MissingCommandlineFlag as exc_: exc = exc_ - self.assertTrue(message in str(exc)) - self.assertTrue(exc is not None) + self.assertIn(message, str(exc)) + self.assertIsNotNone(exc) @mock.patch('certbot._internal.log.post_arg_parse_setup') def test_noninteractive(self, _): @@ -662,11 +691,11 @@ class MainTest(test_util.ConfigTestCase): self._call_no_clientmock(args) os_ver = util.get_os_info_ua() ua = acme_net.call_args[1]["user_agent"] - self.assertTrue(os_ver in ua) + self.assertIn(os_ver, ua) import platform plat = platform.platform() if "linux" in plat.lower(): - self.assertTrue(util.get_os_info_ua() in ua) + self.assertIn(util.get_os_info_ua(), ua) with mock.patch('certbot._internal.main.client.acme_client.ClientNetwork') as acme_net: ua = "bandersnatch" @@ -775,8 +804,8 @@ class MainTest(test_util.ConfigTestCase): # Sending nginx a non-existent conf dir will simulate misconfiguration # (we can only do that if certbot-nginx is actually present) ret, _, _, _ = self._call(args) - self.assertTrue("The nginx plugin is not working" in ret) - self.assertTrue("MisconfigurationError" in ret) + self.assertIn("The nginx plugin is not working", ret) + self.assertIn("MisconfigurationError", ret) self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably") @@ -785,7 +814,7 @@ class MainTest(test_util.ConfigTestCase): mock_gsc.return_value = mock.MagicMock() self._call(["certonly", "--manual", "-d", "foo.bar"]) unused_config, auth, unused_installer = mock_init.call_args[0] - self.assertTrue(isinstance(auth, manual.Authenticator)) + self.assertIsInstance(auth, manual.Authenticator) with mock.patch('certbot._internal.main.certonly') as mock_certonly: self._call(["auth", "--standalone"]) @@ -828,7 +857,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() stdout = io.StringIO() @@ -843,7 +872,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args_unprivileged(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() def throw_error(directory, mode, strict): @@ -865,7 +894,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_init(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() stdout = io.StringIO() @@ -883,7 +912,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_prepare(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() stdout = io.StringIO() @@ -923,7 +952,7 @@ class MainTest(test_util.ConfigTestCase): self._call(['-a', 'bad_auth', 'certonly']) assert False, "Exception should have been raised" except errors.PluginSelectionError as e: - self.assertTrue('The requested bad_auth plugin does not appear' in str(e)) + self.assertIn('The requested bad_auth plugin does not appear', str(e)) def test_check_config_sanity_domain(self): # FQDN @@ -982,8 +1011,7 @@ class MainTest(test_util.ConfigTestCase): self._certonly_new_request_common(mock_client, ['--dry-run']) self.assertEqual( mock_client.obtain_and_enroll_certificate.call_count, 1) - self.assertTrue( - 'dry run' in mock_get_utility().add_message.call_args[0][0]) + self.assertIn('dry run', mock_get_utility().add_message.call_args[0][0]) # Asserts we don't suggest donating after a successful dry run self.assertEqual(mock_get_utility().add_message.call_count, 1) @@ -1004,12 +1032,11 @@ class MainTest(test_util.ConfigTestCase): self.assertEqual( mock_client.obtain_and_enroll_certificate.call_count, 1) cert_msg = mock_get_utility().add_message.call_args_list[0][0][0] - self.assertTrue(cert_path in cert_msg) - self.assertTrue(date in cert_msg) - self.assertTrue(key_path in cert_msg) - self.assertTrue( - 'donate' in mock_get_utility().add_message.call_args[0][0]) - self.assertTrue(mock_subscription.called) + self.assertIn(cert_path, cert_msg) + self.assertIn(date, cert_msg) + self.assertIn(key_path, cert_msg) + self.assertIn('donate', mock_get_utility().add_message.call_args[0][0]) + self.assertIs(mock_subscription.called, True) @mock.patch('certbot._internal.eff.handle_subscription') def test_certonly_new_request_failure(self, mock_subscription): @@ -1017,7 +1044,7 @@ class MainTest(test_util.ConfigTestCase): mock_client.obtain_and_enroll_certificate.return_value = False self.assertRaises(errors.Error, self._certonly_new_request_common, mock_client) - self.assertFalse(mock_subscription.called) + self.assertIs(mock_subscription.called, False) def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, @@ -1053,7 +1080,7 @@ class MainTest(test_util.ConfigTestCase): mock_get_utility().notification.side_effect = write_msg with mock.patch('certbot._internal.main.renewal.OpenSSL') as mock_ssl: mock_latest = mock.MagicMock() - mock_latest.get_issuer.return_value = "Fake fake" + mock_latest.get_issuer.return_value = "Artificial pretend" mock_ssl.crypto.load_certificate.return_value = mock_latest with mock.patch('certbot._internal.main.renewal.crypto_util') \ as mock_crypto_util: @@ -1092,7 +1119,7 @@ class MainTest(test_util.ConfigTestCase): finally: if log_out: with open(os.path.join(self.config.logs_dir, "letsencrypt.log")) as lf: - self.assertTrue(log_out in lf.read()) + self.assertIn(log_out, lf.read()) return mock_lineage, mock_get_utility, stdout @@ -1103,8 +1130,8 @@ class MainTest(test_util.ConfigTestCase): lineage.update_all_links_to.assert_called_once_with( lineage.latest_common_version()) cert_msg = get_utility().add_message.call_args_list[0][0][0] - self.assertTrue('fullchain.pem' in cert_msg) - self.assertTrue('donate' in get_utility().add_message.call_args[0][0]) + self.assertIn('fullchain.pem', cert_msg) + self.assertIn('donate', get_utility().add_message.call_args[0][0]) @mock.patch('certbot._internal.log.logging.handlers.RotatingFileHandler.doRollover') @mock.patch('certbot.crypto_util.notAfter') @@ -1113,7 +1140,7 @@ class MainTest(test_util.ConfigTestCase): _, get_utility, _ = self._test_renewal_common(False, ['--dry-run', '--keep'], log_out="simulating renewal") self.assertEqual(get_utility().add_message.call_count, 1) - self.assertTrue('dry run' in get_utility().add_message.call_args[0][0]) + self.assertIn('dry run', get_utility().add_message.call_args[0][0]) self._test_renewal_common(False, ['--renew-by-default', '-tvv', '--debug'], log_out="Auto-renewal forced") @@ -1172,8 +1199,8 @@ class MainTest(test_util.ConfigTestCase): expiry = datetime.datetime.now() + datetime.timedelta(days=90) _, _, stdout = self._test_renewal_common(False, extra_args=None, should_renew=False, args=['renew'], expiry_date=expiry) - self.assertTrue('No renewals were attempted.' in stdout.getvalue()) - self.assertTrue('The following certificates are not due for renewal yet:' in stdout.getvalue()) + self.assertIn('No renewals were attempted.', stdout.getvalue()) + self.assertIn('The following certificates are not due for renewal yet:', stdout.getvalue()) @mock.patch('certbot._internal.log.post_arg_parse_setup') def test_quiet_renew(self, _): @@ -1181,7 +1208,7 @@ class MainTest(test_util.ConfigTestCase): args = ["renew", "--dry-run"] _, _, stdout = self._test_renewal_common(True, [], args=args, should_renew=True) out = stdout.getvalue() - self.assertTrue("renew" in out) + self.assertIn("renew", out) args = ["renew", "--dry-run", "-q"] _, _, stdout = self._test_renewal_common(True, [], args=args, @@ -1247,7 +1274,7 @@ class MainTest(test_util.ConfigTestCase): if assert_oc_called: self.assertTrue(mock_renew_cert.called) else: - self.assertFalse(mock_renew_cert.called) + self.assertIs(mock_renew_cert.called, False) def test_renew_no_renewalparams(self): self._test_renew_common(assert_oc_called=False, error_expected=True) @@ -1324,9 +1351,9 @@ class MainTest(test_util.ConfigTestCase): _, _, stdout = self._test_renewal_common( due_for_renewal=False, extra_args=None, should_renew=False, args=['renew', '--post-hook', - '{0} -c "from __future__ import print_function; print(\'hello world\');"' + '{0} -c "print(\'hello world\');"' .format(sys.executable)]) - self.assertTrue('No hooks were run.' in stdout.getvalue()) + self.assertIn('No hooks were run.', stdout.getvalue()) @test_util.patch_get_utility() @mock.patch('certbot._internal.main._find_lineage_for_domains_and_certname') @@ -1337,8 +1364,8 @@ class MainTest(test_util.ConfigTestCase): mock_renewal.return_value = ('reinstall', mock.MagicMock()) mock_init.return_value = mock_client = mock.MagicMock() self._call(['-d', 'foo.bar', '-a', 'standalone', 'certonly']) - self.assertFalse(mock_client.obtain_certificate.called) - self.assertFalse(mock_client.obtain_and_enroll_certificate.called) + self.assertIs(mock_client.obtain_certificate.called, False) + self.assertIs(mock_client.obtain_and_enroll_certificate.called, False) self.assertEqual(mock_get_utility().add_message.call_count, 0) mock_report_new_cert.assert_not_called() #self.assertTrue('donate' not in mock_get_utility().add_message.call_args[0][0]) @@ -1370,7 +1397,7 @@ class MainTest(test_util.ConfigTestCase): self._call(args) if '--dry-run' in args: - self.assertFalse(mock_client.save_certificate.called) + self.assertIs(mock_client.save_certificate.called, False) else: mock_client.save_certificate.assert_called_once_with( certr, chain, cert_path, chain_path, full_path) @@ -1381,17 +1408,15 @@ class MainTest(test_util.ConfigTestCase): def test_certonly_csr(self, mock_subscription): mock_get_utility = self._test_certonly_csr_common() cert_msg = mock_get_utility().add_message.call_args_list[0][0][0] - self.assertTrue('fullchain.pem' in cert_msg) - self.assertFalse('Your key file has been saved at' in cert_msg) - self.assertTrue( - 'donate' in mock_get_utility().add_message.call_args[0][0]) - self.assertTrue(mock_subscription.called) + self.assertIn('fullchain.pem', cert_msg) + self.assertNotIn('Your key file has been saved at', cert_msg) + self.assertIn('donate', mock_get_utility().add_message.call_args[0][0]) + self.assertIs(mock_subscription.called, True) def test_certonly_csr_dry_run(self): mock_get_utility = self._test_certonly_csr_common(['--dry-run']) self.assertEqual(mock_get_utility().add_message.call_count, 1) - self.assertTrue( - 'dry run' in mock_get_utility().add_message.call_args[0][0]) + self.assertIn('dry run', mock_get_utility().add_message.call_args[0][0]) @mock.patch('certbot._internal.main._delete_if_appropriate') @mock.patch('certbot._internal.main.client.acme_client') @@ -1446,7 +1471,7 @@ class MainTest(test_util.ConfigTestCase): mocked_account.AccountFileStorage.return_value = mocked_storage mocked_storage.find_all.return_value = ["an account"] x = self._call_no_clientmock(["register", "--email", "user@example.org"]) - self.assertTrue("There is an existing account" in x[0]) + self.assertIn("There is an existing account", x[0]) @mock.patch('certbot._internal.plugins.selection.choose_configurator_plugins') @mock.patch('certbot._internal.updater._run_updaters') @@ -1459,7 +1484,7 @@ class MainTest(test_util.ConfigTestCase): updater.run_generic_updaters(self.config, None, None) # Make sure we're returning None, and hence not trying to run the # without installer - self.assertFalse(mock_run.called) + self.assertIs(mock_run.called, False) class UnregisterTest(unittest.TestCase): @@ -1503,7 +1528,7 @@ class UnregisterTest(unittest.TestCase): res = main.unregister(config, unused_plugins) - self.assertTrue(res is None) + self.assertIsNone(res) mock_notify.assert_called_once_with("Account deactivated.") def test_unregister_no_account(self): @@ -1520,7 +1545,7 @@ class UnregisterTest(unittest.TestCase): res = main.unregister(config, unused_plugins) m = "Could not find existing account to deactivate." self.assertEqual(res, m) - self.assertFalse(cb_client.acme.deactivate_registration.called) + self.assertIs(cb_client.acme.deactivate_registration.called, False) class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): @@ -1548,7 +1573,7 @@ class EnhanceTest(test_util.ConfigTestCase): """Tests for certbot._internal.main.enhance.""" def setUp(self): - super(EnhanceTest, self).setUp() + super().setUp() self.get_utility_patch = test_util.patch_get_utility() self.mock_get_utility = self.get_utility_patch.start() self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) @@ -1584,7 +1609,7 @@ class EnhanceTest(test_util.ConfigTestCase): self._call(['enhance', '--redirect']) self.assertTrue(mock_pick.called) # Check that the message includes "enhancements" - self.assertTrue("enhancements" in mock_pick.call_args[0][3]) + self.assertIn("enhancements", mock_pick.call_args[0][3]) @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins') @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @@ -1598,7 +1623,7 @@ class EnhanceTest(test_util.ConfigTestCase): with mock.patch('certbot._internal.main.plug_sel.logger.warning') as mock_log: mock_client = self._call(['enhance', '-a', 'webroot', '--redirect']) self.assertTrue(mock_log.called) - self.assertTrue("make sense" in mock_log.call_args[0][0]) + self.assertIn("make sense", mock_log.call_args[0][0]) self.assertTrue(mock_client.enhance_config.called) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @@ -1616,8 +1641,8 @@ class EnhanceTest(test_util.ConfigTestCase): all(getattr(mock_client.config, e) for e in req_enh)) self.assertFalse( any(getattr(mock_client.config, e) for e in not_req_enh)) - self.assertTrue( - "example.com" in mock_client.enhance_config.call_args[0][0]) + self.assertIn( + "example.com", mock_client.enhance_config.call_args[0][0]) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @mock.patch('certbot._internal.main.display_ops.choose_values') @@ -1630,7 +1655,7 @@ class EnhanceTest(test_util.ConfigTestCase): mock_client = self._call(['enhance', '--redirect', '--hsts', '--non-interactive']) self.assertTrue(mock_client.enhance_config.called) - self.assertFalse(mock_choose.called) + self.assertIs(mock_choose.called, False) @mock.patch('certbot._internal.main.display_ops.choose_values') @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins') @@ -1653,7 +1678,7 @@ class EnhanceTest(test_util.ConfigTestCase): mock_pick.return_value = (None, None) mock_pick.side_effect = errors.PluginSelectionError() mock_client = self._call(['enhance', '--hsts']) - self.assertFalse(mock_client.enhance_config.called) + self.assertIs(mock_client.enhance_config.called, False) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @mock.patch('certbot._internal.main.display_ops.choose_values') @@ -1692,7 +1717,7 @@ class InstallTest(test_util.ConfigTestCase): """Tests for certbot._internal.main.install.""" def setUp(self): - super(InstallTest, self).setUp() + super().setUp() self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins') @@ -1737,7 +1762,7 @@ class UpdateAccountTest(test_util.ConfigTestCase): for patch in patches.values(): self.addCleanup(patch.stop) - return super(UpdateAccountTest, self).setUp() + return super().setUp() def _call(self, args): with mock.patch('certbot._internal.main.sys.stdout'), \ diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index fe89fff9f..72a39e2d1 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -68,14 +68,14 @@ class OCSPTestOpenSSL(unittest.TestCase): mock_communicate.communicate.return_value = (None, out.partition("\n")[2]) checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(checker.host_args("x"), ["Host", "x"]) - self.assertEqual(checker.broken, False) + self.assertIs(checker.broken, False) mock_exists.return_value = False mock_popen.call_count = 0 checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 0) self.assertEqual(mock_log.call_count, 1) - self.assertEqual(checker.broken, True) + self.assertIs(checker.broken, True) @mock.patch('certbot.ocsp._determine_ocsp_server') @mock.patch('certbot.ocsp.crypto_util.notAfter') @@ -89,24 +89,24 @@ class OCSPTestOpenSSL(unittest.TestCase): self.checker.broken = True mock_determine.return_value = ("", "") - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.checker.broken = False mock_run.return_value = tuple(openssl_happy[1:]) - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.assertEqual(mock_run.call_count, 0) mock_determine.return_value = ("http://x.co", "x.co") - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) mock_run.side_effect = errors.SubprocessError("Unable to load certificate launcher") - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.assertEqual(mock_run.call_count, 2) # cert expired mock_na.return_value = now mock_determine.return_value = ("", "") count_before = mock_determine.call_count - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.assertEqual(mock_determine.call_count, count_before) def test_determine_ocsp_server(self): @@ -122,22 +122,22 @@ class OCSPTestOpenSSL(unittest.TestCase): # pylint: disable=protected-access mock_run.return_value = openssl_confused from certbot import ocsp - self.assertEqual(ocsp._translate_ocsp_query(*openssl_happy), False) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_confused), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_happy), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_confused), False) self.assertEqual(mock_log.debug.call_count, 1) self.assertEqual(mock_log.warning.call_count, 0) mock_log.debug.call_count = 0 - self.assertEqual(ocsp._translate_ocsp_query(*openssl_unknown), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_unknown), False) self.assertEqual(mock_log.debug.call_count, 1) self.assertEqual(mock_log.warning.call_count, 0) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_expired_ocsp), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_expired_ocsp), False) self.assertEqual(mock_log.debug.call_count, 2) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_broken), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_broken), False) self.assertEqual(mock_log.warning.call_count, 1) mock_log.info.call_count = 0 - self.assertEqual(ocsp._translate_ocsp_query(*openssl_revoked), True) + self.assertIs(ocsp._translate_ocsp_query(*openssl_revoked), True) self.assertEqual(mock_log.info.call_count, 0) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_expired_ocsp_revoked), True) + self.assertIs(ocsp._translate_ocsp_query(*openssl_expired_ocsp_revoked), True) self.assertEqual(mock_log.info.call_count, 1) @@ -236,17 +236,17 @@ class OSCPTestCryptography(unittest.TestCase): with _ocsp_mock(ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, http_status_code=400): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OCSP response in invalid with _ocsp_mock(ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.UNAUTHORIZED): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OCSP response is valid, but certificate status is unknown with _ocsp_mock(ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # The OCSP response says that the certificate is revoked, but certificate # does not contain the OCSP extension. @@ -255,32 +255,32 @@ class OSCPTestCryptography(unittest.TestCase): side_effect=x509.ExtensionNotFound( 'Not found', x509.AuthorityInformationAccessOID.OCSP)): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OCSP response uses an unsupported signature. with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, check_signature_side_effect=UnsupportedAlgorithm('foo')): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OSCP signature response is invalid. with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, check_signature_side_effect=InvalidSignature('foo')): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # Assertion error on OCSP response validity with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, check_signature_side_effect=AssertionError('foo')): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # No responder cert in OCSP response with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) as mocks: mocks['mock_response'].return_value.certificates = [] revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # Responder cert is not signed by certificate issuer with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, @@ -289,7 +289,7 @@ class OSCPTestCryptography(unittest.TestCase): mocks['mock_response'].return_value.certificates[0] = mock.Mock( issuer='fake', subject=cert.subject) revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL): # This mock is necessary to avoid the first call contained in _determine_ocsp_server @@ -300,7 +300,7 @@ class OSCPTestCryptography(unittest.TestCase): side_effect=x509.ExtensionNotFound( 'Not found', x509.AuthorityInformationAccessOID.OCSP)): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) @contextlib.contextmanager diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index bcd190e9b..6eb5dbfce 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -88,7 +88,7 @@ class InstallerTest(test_util.ConfigTestCase): """Tests for certbot.plugins.common.Installer.""" def setUp(self): - super(InstallerTest, self).setUp() + super().setUp() filesystem.mkdir(self.config.config_dir) from certbot.plugins.common import Installer @@ -174,7 +174,7 @@ class InstallerTest(test_util.ConfigTestCase): def test_current_file_hash_in_all_hashes(self): from certbot._internal.constants import ALL_SSL_DHPARAMS_HASHES - self.assertTrue(self._current_ssl_dhparams_hash() in ALL_SSL_DHPARAMS_HASHES, + self.assertIn(self._current_ssl_dhparams_hash(), ALL_SSL_DHPARAMS_HASHES, "Constants.ALL_SSL_DHPARAMS_HASHES must be appended" " with the sha256 hash of self.config.ssl_dhparams when it is updated.") @@ -282,7 +282,7 @@ class InstallVersionControlledFileTest(test_util.TempDirTestCase): """Tests for certbot.plugins.common.install_version_controlled_file.""" def setUp(self): - super(InstallVersionControlledFileTest, self).setUp() + super().setUp() self.hashes = ["someotherhash"] self.dest_path = os.path.join(self.tempdir, "options-ssl-dest.conf") self.hash_path = os.path.join(self.tempdir, ".options-ssl-conf.txt") @@ -330,7 +330,7 @@ class InstallVersionControlledFileTest(test_util.TempDirTestCase): mod_ssl_conf.write("a new line for the wrong hash\n") with mock.patch("certbot.plugins.common.logger") as mock_logger: self._call() - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) self.assertTrue(os.path.isfile(self.dest_path)) self.assertEqual(crypto_util.sha256sum(self.source_path), self._current_file_hash()) @@ -352,7 +352,7 @@ class InstallVersionControlledFileTest(test_util.TempDirTestCase): # only print warning once with mock.patch("certbot.plugins.common.logger") as mock_logger: self._call() - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index a97de24b9..83dfb41ca 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -2,6 +2,7 @@ import functools import string import unittest +from typing import List try: import mock @@ -74,7 +75,7 @@ class PluginEntryPointTest(unittest.TestCase): name, PluginEntryPoint.entry_point_to_plugin_name(entry_point, with_prefix=True)) def test_description(self): - self.assertTrue("temporary webserver" in self.plugin_ep.description) + self.assertIn("temporary webserver", self.plugin_ep.description) def test_description_with_name(self): self.plugin_ep.plugin_cls = mock.MagicMock(description="Desc") @@ -100,31 +101,31 @@ class PluginEntryPointTest(unittest.TestCase): interfaces.IInstaller, interfaces.IAuthenticator))) def test__init__(self): - self.assertFalse(self.plugin_ep.initialized) - self.assertFalse(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) - self.assertTrue(self.plugin_ep.problem is None) - self.assertTrue(self.plugin_ep.entry_point is EP_SA) + self.assertIs(self.plugin_ep.initialized, False) + self.assertIs(self.plugin_ep.prepared, False) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) + self.assertIsNone(self.plugin_ep.problem) + self.assertIs(self.plugin_ep.entry_point, EP_SA) self.assertEqual("sa", self.plugin_ep.name) - self.assertTrue(self.plugin_ep.plugin_cls is standalone.Authenticator) + self.assertIs(self.plugin_ep.plugin_cls, standalone.Authenticator) def test_init(self): config = mock.MagicMock() plugin = self.plugin_ep.init(config=config) - self.assertTrue(self.plugin_ep.initialized) - self.assertTrue(plugin.config is config) + self.assertIs(self.plugin_ep.initialized, True) + self.assertIs(plugin.config, config) # memoize! - self.assertTrue(self.plugin_ep.init() is plugin) - self.assertTrue(plugin.config is config) + self.assertIs(self.plugin_ep.init(), plugin) + self.assertIs(plugin.config, config) # try to give different config - self.assertTrue(self.plugin_ep.init(123) is plugin) - self.assertTrue(plugin.config is config) + self.assertIs(self.plugin_ep.init(123), plugin) + self.assertIs(plugin.config, config) - self.assertFalse(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) + self.assertIs(self.plugin_ep.prepared, False) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) def test_verify(self): iface1 = mock.MagicMock(__name__="iface1") @@ -154,7 +155,7 @@ class PluginEntryPointTest(unittest.TestCase): self.plugin_ep.init(config=config) self.plugin_ep.prepare() self.assertTrue(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) + self.assertIs(self.plugin_ep.misconfigured, False) # output doesn't matter that much, just test if it runs str(self.plugin_ep) @@ -164,12 +165,11 @@ class PluginEntryPointTest(unittest.TestCase): plugin.prepare.side_effect = errors.MisconfigurationError # pylint: disable=protected-access self.plugin_ep._initialized = plugin - self.assertTrue(isinstance(self.plugin_ep.prepare(), - errors.MisconfigurationError)) + self.assertIsInstance(self.plugin_ep.prepare(), + errors.MisconfigurationError) self.assertTrue(self.plugin_ep.prepared) self.assertTrue(self.plugin_ep.misconfigured) - self.assertTrue(isinstance(self.plugin_ep.problem, - errors.MisconfigurationError)) + self.assertIsInstance(self.plugin_ep.problem, errors.MisconfigurationError) self.assertTrue(self.plugin_ep.available) def test_prepare_no_installation(self): @@ -177,21 +177,20 @@ class PluginEntryPointTest(unittest.TestCase): plugin.prepare.side_effect = errors.NoInstallationError # pylint: disable=protected-access self.plugin_ep._initialized = plugin - self.assertTrue(isinstance(self.plugin_ep.prepare(), - errors.NoInstallationError)) - self.assertTrue(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) + self.assertIsInstance(self.plugin_ep.prepare(), errors.NoInstallationError) + self.assertIs(self.plugin_ep.prepared, True) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) def test_prepare_generic_plugin_error(self): plugin = mock.MagicMock() plugin.prepare.side_effect = errors.PluginError # pylint: disable=protected-access self.plugin_ep._initialized = plugin - self.assertTrue(isinstance(self.plugin_ep.prepare(), errors.PluginError)) + self.assertIsInstance(self.plugin_ep.prepare(), errors.PluginError) self.assertTrue(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) def test_repr(self): self.assertEqual("PluginEntryPoint#sa", repr(self.plugin_ep)) @@ -225,14 +224,14 @@ class PluginsRegistryTest(unittest.TestCase): standalone.Authenticator, webroot.Authenticator, null.Installer, null.Installer] plugins = PluginsRegistry.find_all() - self.assertTrue(plugins["sa"].plugin_cls is standalone.Authenticator) - self.assertTrue(plugins["sa"].entry_point is EP_SA) - self.assertTrue(plugins["wr"].plugin_cls is webroot.Authenticator) - self.assertTrue(plugins["wr"].entry_point is EP_WR) - self.assertTrue(plugins["ep1"].plugin_cls is null.Installer) - self.assertTrue(plugins["ep1"].entry_point is self.ep1) - self.assertTrue(plugins["p1:ep1"].plugin_cls is null.Installer) - self.assertTrue(plugins["p1:ep1"].entry_point is self.ep1) + self.assertIs(plugins["sa"].plugin_cls, standalone.Authenticator) + self.assertIs(plugins["sa"].entry_point, EP_SA) + self.assertIs(plugins["wr"].plugin_cls, webroot.Authenticator) + self.assertIs(plugins["wr"].entry_point, EP_WR) + self.assertIs(plugins["ep1"].plugin_cls, null.Installer) + self.assertIs(plugins["ep1"].entry_point, self.ep1) + self.assertIs(plugins["p1:ep1"].plugin_cls, null.Installer) + self.assertIs(plugins["p1:ep1"].entry_point, self.ep1) def test_getitem(self): self.assertEqual(self.plugin_ep, self.reg["mock"]) @@ -277,7 +276,7 @@ class PluginsRegistryTest(unittest.TestCase): self.plugin_ep.prepare.assert_called_once_with() def test_prepare_order(self): - order = [] # type: List[str] + order: List[str] = [] plugins = dict( (c, mock.MagicMock(prepare=functools.partial(order.append, c))) for c in string.ascii_letters) @@ -295,10 +294,10 @@ class PluginsRegistryTest(unittest.TestCase): self.assertEqual({}, self.reg.available()._plugins) def test_find_init(self): - self.assertTrue(self.reg.find_init(mock.Mock()) is None) + self.assertIsNone(self.reg.find_init(mock.Mock())) self.plugin_ep.initialized = True - self.assertTrue( - self.reg.find_init(self.plugin_ep.init()) is self.plugin_ep) + self.assertIs( + self.reg.find_init(self.plugin_ep.init()), self.plugin_ep) def test_repr(self): self.plugin_ep.__repr__ = lambda _: "PluginEntryPoint#mock" diff --git a/certbot/tests/plugins/dns_common_lexicon_test.py b/certbot/tests/plugins/dns_common_lexicon_test.py index a67430f3e..40afd107b 100644 --- a/certbot/tests/plugins/dns_common_lexicon_test.py +++ b/certbot/tests/plugins/dns_common_lexicon_test.py @@ -17,7 +17,7 @@ class LexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconCl pass def setUp(self): - super(LexiconClientTest, self).setUp() + super().setUp() self.client = LexiconClientTest._FakeLexiconClient() self.provider_mock = mock.MagicMock() diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index 993f3b461..12fac00d0 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -29,14 +29,14 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen def more_info(self): # pylint: disable=missing-docstring,no-self-use return 'A fake authenticator for testing.' - class _FakeConfig(object): + class _FakeConfig: fake_propagation_seconds = 0 fake_config_key = 1 fake_other_key = None fake_file_path = None def setUp(self): - super(DNSAuthenticatorTest, self).setUp() + super().setUp() self.config = DNSAuthenticatorTest._FakeConfig() @@ -164,7 +164,7 @@ class CredentialsConfigurationTest(test_util.TempDirTestCase): class CredentialsConfigurationRequireTest(test_util.TempDirTestCase): def setUp(self): - super(CredentialsConfigurationRequireTest, self).setUp() + super().setUp() self.path = os.path.join(self.tempdir, 'file.ini') @@ -211,20 +211,20 @@ class CredentialsConfigurationRequireTest(test_util.TempDirTestCase): class DomainNameGuessTest(unittest.TestCase): def test_simple_case(self): - self.assertTrue( - 'example.com' in + self.assertIn( + 'example.com', dns_common.base_domain_name_guesses("example.com") ) def test_sub_domain(self): - self.assertTrue( - 'example.com' in + self.assertIn( + 'example.com', dns_common.base_domain_name_guesses("foo.bar.baz.example.com") ) def test_second_level_domain(self): - self.assertTrue( - 'example.co.uk' in + self.assertIn( + 'example.co.uk', dns_common.base_domain_name_guesses("foo.bar.baz.example.co.uk") ) diff --git a/certbot/tests/plugins/enhancements_test.py b/certbot/tests/plugins/enhancements_test.py index a20a6864f..0aa1512b4 100644 --- a/certbot/tests/plugins/enhancements_test.py +++ b/certbot/tests/plugins/enhancements_test.py @@ -15,7 +15,7 @@ class EnhancementTest(test_util.ConfigTestCase): """Tests for new style enhancements in certbot.plugins.enhancements""" def setUp(self): - super(EnhancementTest, self).setUp() + super().setUp() self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index f3c580517..c97e08fa6 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -19,7 +19,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): """Tests for certbot._internal.plugins.manual.Authenticator.""" def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() self.http_achall = acme_util.HTTP01_A self.dns_achall = acme_util.DNS01_A self.dns_achall_2 = acme_util.DNS01_A_2 @@ -52,7 +52,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.assertRaises(errors.HookCommandNotFound, self.auth.prepare) def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), str)) + self.assertIsInstance(self.auth.more_info(), str) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref('example.org'), @@ -60,7 +60,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): def test_script_perform(self): self.config.manual_auth_hook = ( - '{0} -c "from __future__ import print_function;' + '{0} -c "' 'from certbot.compat import os;' 'print(os.environ.get(\'CERTBOT_DOMAIN\'));' 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' @@ -96,9 +96,8 @@ class AuthenticatorTest(test_util.TempDirTestCase): [achall.response(achall.account_key) for achall in self.achalls]) for i, (args, kwargs) in enumerate(mock_get_utility().notification.call_args_list): achall = self.achalls[i] - self.assertTrue( - achall.validation(achall.account_key) in args[0]) - self.assertFalse(kwargs['wrap']) + self.assertIn(achall.validation(achall.account_key), args[0]) + self.assertIs(kwargs['wrap'], False) def test_cleanup(self): self.config.manual_auth_hook = ('{0} -c "import sys; sys.stdout.write(\'foo\')"' @@ -119,7 +118,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): os.environ['CERTBOT_TOKEN'], achall.chall.encode('token')) else: - self.assertFalse('CERTBOT_TOKEN' in os.environ) + self.assertNotIn('CERTBOT_TOKEN', os.environ) if __name__ == '__main__': diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index dad5b270a..dfdd0a7de 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -15,7 +15,7 @@ class InstallerTest(unittest.TestCase): self.installer = Installer(config=mock.MagicMock(), name="null") def test_it(self): - self.assertTrue(isinstance(self.installer.more_info(), str)) + self.assertIsInstance(self.installer.more_info(), str) self.assertEqual([], self.installer.get_all_names()) self.assertEqual([], self.installer.supported_enhancements()) diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index a5de99e60..60917626a 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -1,6 +1,7 @@ """Tests for letsencrypt.plugins.selection""" import sys import unittest +from typing import List try: import mock @@ -52,7 +53,7 @@ class PickPluginTest(unittest.TestCase): self.default = None self.reg = mock.MagicMock() self.question = "Question?" - self.ifaces = [] # type: List[interfaces.IPlugin] + self.ifaces: List[interfaces.IPlugin] = [] def _call(self): from certbot._internal.plugins.selection import pick_plugin @@ -69,7 +70,7 @@ class PickPluginTest(unittest.TestCase): self.assertEqual(1, self.reg.visible().ifaces.call_count) def test_no_candidate(self): - self.assertTrue(self._call() is None) + self.assertIsNone(self._call()) def test_single(self): plugin_ep = mock.MagicMock() @@ -87,7 +88,7 @@ class PickPluginTest(unittest.TestCase): self.reg.visible().ifaces().verify().available.return_value = { "bar": plugin_ep} - self.assertTrue(self._call() is None) + self.assertIsNone(self._call()) def test_multiple(self): plugin_ep = mock.MagicMock() @@ -110,7 +111,7 @@ class PickPluginTest(unittest.TestCase): with mock.patch("certbot._internal.plugins.selection.choose_plugin") as mock_choose: mock_choose.return_value = None - self.assertTrue(self._call() is None) + self.assertIsNone(self._call()) class ChoosePluginTest(unittest.TestCase): @@ -152,34 +153,14 @@ class ChoosePluginTest(unittest.TestCase): @test_util.patch_get_utility("certbot._internal.plugins.selection.z_util") def test_no_choice(self, mock_util): mock_util().menu.return_value = (display_util.CANCEL, 0) - self.assertTrue(self._call() is None) - - @test_util.patch_get_utility("certbot._internal.plugins.selection.z_util") - def test_new_interaction_avoidance(self, mock_util): - mock_nginx = mock.Mock( - description_with_name="n", misconfigured=False) - mock_nginx.init().more_info.return_value = "nginx plugin" - mock_nginx.name = "nginx" - self.plugins[1] = mock_nginx - mock_util().menu.return_value = (display_util.CANCEL, 0) - - unset_cb_auto = os.environ.get("CERTBOT_AUTO") is None - if unset_cb_auto: - os.environ["CERTBOT_AUTO"] = "foo" - try: - self._call() - finally: - if unset_cb_auto: - del os.environ["CERTBOT_AUTO"] - - self.assertTrue("default" in mock_util().menu.call_args[1]) + self.assertIsNone(self._call()) class GetUnpreparedInstallerTest(test_util.ConfigTestCase): """Tests for certbot._internal.plugins.selection.get_unprepared_installer.""" def setUp(self): - super(GetUnpreparedInstallerTest, self).setUp() + super().setUp() self.mock_apache_fail_ep = mock.Mock( description_with_name="afail") self.mock_apache_fail_ep.check_name = lambda name: name == "afail" @@ -199,7 +180,7 @@ class GetUnpreparedInstallerTest(test_util.ConfigTestCase): def test_no_installer_defined(self): self.config.configurator = None - self.assertEqual(self._call(), None) + self.assertIsNone(self._call()) def test_no_available_installers(self): self.config.configurator = "apache" @@ -209,7 +190,7 @@ class GetUnpreparedInstallerTest(test_util.ConfigTestCase): def test_get_plugin(self): self.config.configurator = "apache" installer = self._call() - self.assertTrue(installer is self.mock_apache_plugin) + self.assertIs(installer, self.mock_apache_plugin) def test_multiple_installers_returned(self): self.config.configurator = "apache" diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 596cee622..6f2ae91ba 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -1,8 +1,8 @@ """Tests for certbot._internal.plugins.standalone.""" -# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi +import errno import socket -from socket import errno as socket_errors # type: ignore import unittest +from typing import Dict, Set, Tuple import josepy as jose try: @@ -24,15 +24,13 @@ class ServerManagerTest(unittest.TestCase): def setUp(self): from certbot._internal.plugins.standalone import ServerManager - self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] - self.http_01_resources = {} \ - # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] + self.certs: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] = {} + self.http_01_resources: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] = {} self.mgr = ServerManager(self.certs, self.http_01_resources) def test_init(self): - self.assertTrue(self.mgr.certs is self.certs) - self.assertTrue( - self.mgr.http_01_resources is self.http_01_resources) + self.assertIs(self.mgr.certs, self.certs) + self.assertIs(self.mgr.http_01_resources, self.http_01_resources) def _test_run_stop(self, challenge_type): server = self.mgr.run(port=0, challenge_type=challenge_type) @@ -49,7 +47,7 @@ class ServerManagerTest(unittest.TestCase): port = server.getsocknames()[0][1] server2 = self.mgr.run(port=port, challenge_type=challenges.HTTP01) self.assertEqual(self.mgr.running(), {port: server}) - self.assertTrue(server is server2) + self.assertIs(server, server2) self.mgr.stop(port) self.assertEqual(self.mgr.running(), {}) @@ -90,7 +88,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.servers = mock.MagicMock() def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), str)) + self.assertIsInstance(self.auth.more_info(), str) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref(domain=None), @@ -106,8 +104,8 @@ class AuthenticatorTest(unittest.TestCase): @test_util.patch_get_utility() def test_perform_eaddrinuse_retry(self, mock_get_utility): mock_utility = mock_get_utility() - errno = socket_errors.EADDRINUSE - error = errors.StandaloneBindError(mock.MagicMock(errno=errno), -1) + encountered_errno = errno.EADDRINUSE + error = errors.StandaloneBindError(mock.MagicMock(errno=encountered_errno), -1) self.auth.servers.run.side_effect = [error] + 2 * [mock.MagicMock()] mock_yesno = mock_utility.yesno mock_yesno.return_value = True @@ -121,26 +119,26 @@ class AuthenticatorTest(unittest.TestCase): mock_yesno = mock_utility.yesno mock_yesno.return_value = False - errno = socket_errors.EADDRINUSE - self.assertRaises(errors.PluginError, self._fail_perform, errno) + encountered_errno = errno.EADDRINUSE + self.assertRaises(errors.PluginError, self._fail_perform, encountered_errno) self._assert_correct_yesno_call(mock_yesno) def _assert_correct_yesno_call(self, mock_yesno): yesno_args, yesno_kwargs = mock_yesno.call_args - self.assertTrue("in use" in yesno_args[0]) + self.assertIn("in use", yesno_args[0]) self.assertFalse(yesno_kwargs.get("default", True)) def test_perform_eacces(self): - errno = socket_errors.EACCES - self.assertRaises(errors.PluginError, self._fail_perform, errno) + encountered_errno = errno.EACCES + self.assertRaises(errors.PluginError, self._fail_perform, encountered_errno) def test_perform_unexpected_socket_error(self): - errno = socket_errors.ENOTCONN + encountered_errno = errno.ENOTCONN self.assertRaises( - errors.StandaloneBindError, self._fail_perform, errno) + errors.StandaloneBindError, self._fail_perform, encountered_errno) - def _fail_perform(self, errno): - error = errors.StandaloneBindError(mock.MagicMock(errno=errno), -1) + def _fail_perform(self, encountered_errno): + error = errors.StandaloneBindError(mock.MagicMock(errno=encountered_errno), -1) self.auth.servers.run.side_effect = error self.auth.perform(self._get_achalls()) diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index 2999d306e..d01845510 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -18,7 +18,7 @@ class PluginStorageTest(test_util.ConfigTestCase): """Test for certbot.plugins.storage.PluginStorage""" def setUp(self): - super(PluginStorageTest, self).setUp() + super().setUp() self.plugin_cls = common.Installer filesystem.mkdir(self.config.config_dir) with mock.patch("certbot.reverter.util"): @@ -31,7 +31,7 @@ class PluginStorageTest(test_util.ConfigTestCase): # When unable to read file that exists mock_open = mock.mock_open() mock_open.side_effect = IOError - self.plugin.storage.storagepath = os.path.join(self.config.config_dir, + self.plugin.storage._storagepath = os.path.join(self.config.config_dir, ".pluginstorage.json") with mock.patch("builtins.open", mock_open): with mock.patch('certbot.compat.os.path.isfile', return_value=True): @@ -49,7 +49,7 @@ class PluginStorageTest(test_util.ConfigTestCase): self.assertRaises(KeyError, nocontent.storage.fetch, "value") self.assertTrue(mock_log.called) - self.assertTrue("no values loaded" in mock_log.call_args[0][0]) + self.assertIn("no values loaded", mock_log.call_args[0][0]) def test_load_errors_corrupted(self): with open(os.path.join(self.config.config_dir, @@ -61,17 +61,17 @@ class PluginStorageTest(test_util.ConfigTestCase): self.assertRaises(errors.PluginError, corrupted.storage.fetch, "value") - self.assertTrue("is corrupted" in mock_log.call_args[0][0]) + self.assertIn("is corrupted", mock_log.call_args[0][0]) def test_save_errors_cant_serialize(self): with mock.patch("certbot.plugins.storage.logger.error") as mock_log: # Set data as something that can't be serialized self.plugin.storage._initialized = True # pylint: disable=protected-access - self.plugin.storage.storagepath = "/tmp/whatever" + self.plugin.storage._storagepath = "/tmp/whatever" self.plugin.storage._data = self.plugin_cls # pylint: disable=protected-access self.assertRaises(errors.PluginStorageError, self.plugin.storage.save) - self.assertTrue("Could not serialize" in mock_log.call_args[0][0]) + self.assertIn("Could not serialize", mock_log.call_args[0][0]) def test_save_errors_unable_to_write_file(self): mock_open = mock.mock_open() @@ -80,9 +80,10 @@ class PluginStorageTest(test_util.ConfigTestCase): with mock.patch("certbot.plugins.storage.logger.error") as mock_log: self.plugin.storage._data = {"valid": "data"} # pylint: disable=protected-access self.plugin.storage._initialized = True # pylint: disable=protected-access + self.plugin.storage._storagepath = "/tmp/whatever" self.assertRaises(errors.PluginStorageError, self.plugin.storage.save) - self.assertTrue("Could not write" in mock_log.call_args[0][0]) + self.assertIn("Could not write", mock_log.call_args[0][0]) def test_save_uninitialized(self): with mock.patch("certbot.reverter.util"): @@ -113,7 +114,7 @@ class PluginStorageTest(test_util.ConfigTestCase): ".pluginstorage.json"), 'r') as fh: psdata = fh.read() psjson = json.loads(psdata) - self.assertTrue("mockplugin" in psjson.keys()) + self.assertIn("mockplugin", psjson.keys()) self.assertEqual(len(psjson), 1) self.assertEqual(psjson["mockplugin"]["testkey"], "testvalue") diff --git a/certbot/tests/plugins/util_test.py b/certbot/tests/plugins/util_test.py index 9387b4ae7..1b4fcd652 100644 --- a/certbot/tests/plugins/util_test.py +++ b/certbot/tests/plugins/util_test.py @@ -30,7 +30,7 @@ class PathSurgeryTest(unittest.TestCase): with mock.patch.dict('os.environ', all_path): with mock.patch('certbot.util.exe_exists') as mock_exists: mock_exists.return_value = True - self.assertEqual(path_surgery("eg"), True) + self.assertIs(path_surgery("eg"), True) self.assertEqual(mock_debug.call_count, 0) self.assertEqual(os.environ["PATH"], all_path["PATH"]) if os.name != 'nt': @@ -39,9 +39,9 @@ class PathSurgeryTest(unittest.TestCase): with mock.patch.dict('os.environ', no_path): path_surgery("thingy") self.assertEqual(mock_debug.call_count, 2 if os.name != 'nt' else 1) - self.assertTrue("Failed to find" in mock_debug.call_args[0][0]) - self.assertTrue("/usr/local/bin" in os.environ["PATH"]) - self.assertTrue("/tmp" in os.environ["PATH"]) + self.assertIn("Failed to find", mock_debug.call_args[0][0]) + self.assertIn("/usr/local/bin", os.environ["PATH"]) + self.assertIn("/tmp", os.environ["PATH"]) if __name__ == "__main__": diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index e6fbd8e88..f158486b6 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -58,8 +58,8 @@ class AuthenticatorTest(unittest.TestCase): def test_more_info(self): more_info = self.auth.more_info() - self.assertTrue(isinstance(more_info, str)) - self.assertTrue(self.path in more_info) + self.assertIsInstance(more_info, str) + self.assertIn(self.path, more_info) def test_add_parser_arguments(self): add = mock.MagicMock() @@ -79,7 +79,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.perform([self.achall]) self.assertTrue(mock_display.menu.called) for call in mock_display.menu.call_args_list: - self.assertTrue(self.achall.domain in call[0][0]) + self.assertIn(self.achall.domain, call[0][0]) self.assertTrue(all( webroot in call[0][1] for webroot in self.config.webroot_map.values())) @@ -96,7 +96,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertRaises(errors.PluginError, self.auth.perform, [self.achall]) self.assertTrue(mock_display.menu.called) for call in mock_display.menu.call_args_list: - self.assertTrue(self.achall.domain in call[0][0]) + self.assertIn(self.achall.domain, call[0][0]) self.assertTrue(all( webroot in call[0][1] for webroot in self.config.webroot_map.values())) @@ -141,7 +141,8 @@ class AuthenticatorTest(unittest.TestCase): f.write("thingimy") filesystem.chmod(self.path, 0o000) try: - open(permission_canary, "r") + with open(permission_canary, "r"): + pass print("Warning, running tests as root skips permissions tests...") except IOError: # ok, permissions work, test away... diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index 7c9c53fb4..edee8df6c 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -115,7 +115,7 @@ class RenewalTest(test_util.ConfigTestCase): renewal_candidate = renewal._reconstitute(lineage_config, rc_path) # This means that manual_public_ip_logging_ok was not modified in the config based on its # value in the renewal conf file - self.assertTrue(isinstance(lineage_config.manual_public_ip_logging_ok, mock.MagicMock)) + self.assertIsInstance(lineage_config.manual_public_ip_logging_ok, mock.MagicMock) class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): @@ -129,7 +129,7 @@ class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): def test_allow_subset_of_names_success(self, mock_set_by_cli): mock_set_by_cli.return_value = False self._call(self.config, {'allow_subset_of_names': 'True'}) - self.assertTrue(self.config.allow_subset_of_names is True) + self.assertIs(self.config.allow_subset_of_names, True) @mock.patch('certbot._internal.renewal.cli.set_by_cli') def test_allow_subset_of_names_failure(self, mock_set_by_cli): @@ -164,7 +164,7 @@ class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): def test_must_staple_success(self, mock_set_by_cli): mock_set_by_cli.return_value = False self._call(self.config, {'must_staple': 'True'}) - self.assertTrue(self.config.must_staple is True) + self.assertIs(self.config.must_staple, True) @mock.patch('certbot._internal.renewal.cli.set_by_cli') def test_must_staple_failure(self, mock_set_by_cli): diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index b5ecddb5a..55faff2a4 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -17,7 +17,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): """Tests for interfaces.RenewDeployer and interfaces.GenericUpdater""" def setUp(self): - super(RenewUpdaterTest, self).setUp() + super().setUp() self.generic_updater = mock.MagicMock(spec=interfaces.GenericUpdater) self.generic_updater.restart = mock.MagicMock() self.renew_deployer = mock.MagicMock(spec=interfaces.RenewDeployer) @@ -42,7 +42,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): mock_generic_updater.generic_updates.reset_mock() updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertEqual(mock_generic_updater.generic_updates.call_count, 1) - self.assertFalse(mock_generic_updater.restart.called) + self.assertIs(mock_generic_updater.restart.called, False) def test_renew_deployer(self): lineage = mock.MagicMock() @@ -83,13 +83,13 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.config.disable_renew_updates = True mock_geti.return_value = self.mockinstaller updater.run_generic_updaters(self.config, mock.MagicMock(), None) - self.assertFalse(self.mockinstaller.update_autohsts.called) + self.assertIs(self.mockinstaller.update_autohsts.called, False) def test_enhancement_deployer_not_called(self): self.config.disable_renew_updates = True updater.run_renewal_deployer(self.config, mock.MagicMock(), self.mockinstaller) - self.assertFalse(self.mockinstaller.deploy_autohsts.called) + self.assertIs(self.mockinstaller.deploy_autohsts.called, False) @mock.patch('certbot._internal.plugins.selection.get_unprepared_installer') def test_enhancement_no_updater(self, mock_geti): @@ -105,7 +105,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): mock_geti.return_value = self.mockinstaller with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): updater.run_generic_updaters(self.config, mock.MagicMock(), None) - self.assertFalse(self.mockinstaller.update_autohsts.called) + self.assertIs(self.mockinstaller.update_autohsts.called, False) def test_enhancement_no_deployer(self): FAKEINDEX = [ @@ -120,7 +120,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): updater.run_renewal_deployer(self.config, mock.MagicMock(), self.mockinstaller) - self.assertFalse(self.mockinstaller.deploy_autohsts.called) + self.assertIs(self.mockinstaller.deploy_autohsts.called, False) if __name__ == '__main__': diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 7a37f782e..c4af40591 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -26,8 +26,8 @@ class ReporterTest(unittest.TestCase): self.reporter.add_message("Line 1\nLine 2", self.reporter.LOW_PRIORITY) self.reporter.print_messages() output = sys.stdout.getvalue() # type: ignore - self.assertTrue("Line 1\n" in output) - self.assertTrue("Line 2" in output) + self.assertIn("Line 1\n", output) + self.assertIn("Line 2", output) def test_tty_print_empty(self): sys.stdout.isatty = lambda: True # type: ignore @@ -60,10 +60,10 @@ class ReporterTest(unittest.TestCase): self._add_messages() self.reporter.print_messages() output = sys.stdout.getvalue() # type: ignore - self.assertTrue("IMPORTANT NOTES:" in output) - self.assertTrue("High" in output) - self.assertTrue("Med" in output) - self.assertTrue("Low" in output) + self.assertIn("IMPORTANT NOTES:", output) + self.assertIn("High", output) + self.assertIn("Med", output) + self.assertIn("Low", output) def _unsuccessful_exit_common(self): self._add_messages() @@ -72,10 +72,10 @@ class ReporterTest(unittest.TestCase): except ValueError: self.reporter.print_messages() output = sys.stdout.getvalue() # type: ignore - self.assertTrue("IMPORTANT NOTES:" in output) - self.assertTrue("High" in output) - self.assertTrue("Med" not in output) - self.assertTrue("Low" not in output) + self.assertIn("IMPORTANT NOTES:", output) + self.assertIn("High", output) + self.assertNotIn("Med", output) + self.assertNotIn("Low", output) def _add_messages(self): self.reporter.add_message("High", self.reporter.HIGH_PRIORITY) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index af01a9a1b..e8d85d4d1 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -18,7 +18,7 @@ from certbot.tests import util as test_util class ReverterCheckpointLocalTest(test_util.ConfigTestCase): """Test the Reverter Class.""" def setUp(self): - super(ReverterCheckpointLocalTest, self).setUp() + super().setUp() from certbot.reverter import Reverter # Disable spurious errors... we are trying to test for them @@ -48,7 +48,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): no_change = os.path.join(self.reverter.config.backup_dir, path, "CHANGES_SINCE") with open(no_change, "r") as f: x = f.read() - self.assertTrue("No changes" in x) + self.assertIn("No changes", x) def test_basic_add_to_temp_checkpoint(self): # These shouldn't conflict even though they are both named config.txt @@ -280,7 +280,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): class TestFullCheckpointsReverter(test_util.ConfigTestCase): """Tests functions having to deal with full checkpoints.""" def setUp(self): - super(TestFullCheckpointsReverter, self).setUp() + super().setUp() from certbot.reverter import Reverter # Disable spurious errors... logging.disable(logging.CRITICAL) @@ -328,8 +328,8 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): # One dir left... check title all_dirs = os.listdir(self.config.backup_dir) self.assertEqual(len(all_dirs), 1) - self.assertTrue( - "First Checkpoint" in get_save_notes( + self.assertIn( + "First Checkpoint", get_save_notes( os.path.join(self.config.backup_dir, all_dirs[0]))) # Final rollback self.reverter.rollback_checkpoints(1) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index abd496c8d..aa5910f1e 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -99,7 +99,7 @@ class BaseRenewableCertTest(test_util.ConfigTestCase): def setUp(self): from certbot._internal import storage - super(BaseRenewableCertTest, self).setUp() + super().setUp() # TODO: maybe provide NamespaceConfig.make_dirs? # TODO: main() should create those dirs, c.f. #902 @@ -195,11 +195,11 @@ class RenewableCertTests(BaseRenewableCertTest): from certbot._internal import storage self._write_out_ex_kinds() - self.assertTrue("version" not in self.config_file) + self.assertNotIn("version", self.config_file) with mock.patch("certbot._internal.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) def test_renewal_newer_version(self): from certbot._internal import storage @@ -211,7 +211,7 @@ class RenewableCertTests(BaseRenewableCertTest): with mock.patch("certbot._internal.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) self.assertTrue(mock_logger.info.called) - self.assertTrue("version" in mock_logger.info.call_args[0][0]) + self.assertIn("version", mock_logger.info.call_args[0][0]) def test_consistent(self): # pylint: disable=protected-access @@ -282,7 +282,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version("cert"), 10) def test_no_current_version(self): - self.assertEqual(self.test_rc.current_version("cert"), None) + self.assertIsNone(self.test_rc.current_version("cert")) def test_latest_and_next_versions(self): for ver in range(1, 6): @@ -314,12 +314,12 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.latest_common_version = mock.Mock() mock_has_pending.return_value = False - self.assertEqual(self.test_rc.ensure_deployed(), True) + self.assertIs(self.test_rc.ensure_deployed(), True) self.assertEqual(mock_update.call_count, 0) self.assertEqual(mock_logger.warning.call_count, 0) mock_has_pending.return_value = True - self.assertEqual(self.test_rc.ensure_deployed(), False) + self.assertIs(self.test_rc.ensure_deployed(), False) self.assertEqual(mock_update.call_count, 1) self.assertEqual(mock_logger.warning.call_count, 1) @@ -586,7 +586,7 @@ class RenewableCertTests(BaseRenewableCertTest): self._write_out_kind(kind, 1) self.test_rc.update_all_links_to(1) self.test_rc.save_successor(1, b"newcert", None, b"new chain", self.config) - self.assertFalse(mock_ownership.called) + self.assertIs(mock_ownership.called, False) self.test_rc.save_successor(2, b"newcert", b"new_privkey", b"new chain", self.config) self.assertTrue(mock_ownership.called) @@ -765,18 +765,25 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(storage.add_time_interval(base_time, interval), excepted) + def test_server(self): + self.test_rc.configuration["renewalparams"] = {} + self.assertIsNone(self.test_rc.server) + rp = self.test_rc.configuration["renewalparams"] + rp["server"] = "https://acme.example/dir" + self.assertEqual(self.test_rc.server, "https://acme.example/dir") + def test_is_test_cert(self): self.test_rc.configuration["renewalparams"] = {} rp = self.test_rc.configuration["renewalparams"] - self.assertEqual(self.test_rc.is_test_cert, False) + self.assertIs(self.test_rc.is_test_cert, False) rp["server"] = "https://acme-staging-v02.api.letsencrypt.org/directory" - self.assertEqual(self.test_rc.is_test_cert, True) + self.assertIs(self.test_rc.is_test_cert, True) rp["server"] = "https://staging.someotherca.com/directory" - self.assertEqual(self.test_rc.is_test_cert, True) + self.assertIs(self.test_rc.is_test_cert, True) rp["server"] = "https://acme-v01.api.letsencrypt.org/directory" - self.assertEqual(self.test_rc.is_test_cert, False) + self.assertIs(self.test_rc.is_test_cert, False) rp["server"] = "https://acme-v02.api.letsencrypt.org/directory" - self.assertEqual(self.test_rc.is_test_cert, False) + self.assertIs(self.test_rc.is_test_cert, False) def test_missing_cert(self): from certbot._internal import storage @@ -810,13 +817,13 @@ class RenewableCertTests(BaseRenewableCertTest): with open(temp2, "r") as f: content = f.read() # useful value was updated - self.assertTrue("useful = new_value" in content) + self.assertIn("useful = new_value", content) # associated comment was preserved - self.assertTrue("A useful value" in content) + self.assertIn("A useful value", content) # useless value was deleted - self.assertTrue("useless" not in content) + self.assertNotIn("useless", content) # check version was stored - self.assertTrue("version = {0}".format(certbot.__version__) in content) + self.assertIn("version = {0}".format(certbot.__version__), content) # ensure permissions are copied self.assertEqual(stat.S_IMODE(os.lstat(temp).st_mode), stat.S_IMODE(os.lstat(temp2).st_mode)) @@ -839,7 +846,7 @@ class RenewableCertTests(BaseRenewableCertTest): class DeleteFilesTest(BaseRenewableCertTest): """Tests for certbot._internal.storage.delete_files""" def setUp(self): - super(DeleteFilesTest, self).setUp() + super().setUp() for kind in ALL_FOUR: kind_path = os.path.join(self.config.config_dir, "live", "example.org", @@ -929,19 +936,19 @@ class DeleteFilesTest(BaseRenewableCertTest): class CertPathForCertNameTest(BaseRenewableCertTest): """Test for certbot._internal.storage.cert_path_for_cert_name""" def setUp(self): - super(CertPathForCertNameTest, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config, certname): from certbot._internal.storage import cert_path_for_cert_name return cert_path_for_cert_name(cli_config, certname) def test_simple_cert_name(self): - self.assertEqual(self._call(self.config, 'example.org'), (self.fullchain, 'fullchain')) + self.assertEqual(self._call(self.config, 'example.org'), self.fullchain) def test_no_such_cert_name(self): self.assertRaises(errors.CertStorageError, self._call, self.config, 'fake-example.org') diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 18947c342..a78b614cf 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -108,7 +108,7 @@ class LockDirUntilExit(test_util.TempDirTestCase): return lock_dir_until_exit(*args, **kwargs) def setUp(self): - super(LockDirUntilExit, self).setUp() + super().setUp() # reset global state from other tests import certbot.util reload_module(certbot.util) @@ -164,7 +164,7 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase): """ def setUp(self): - super(MakeOrVerifyDirTest, self).setUp() + super().setUp() self.path = os.path.join(self.tempdir, "foo") filesystem.mkdir(self.path, 0o600) @@ -196,7 +196,7 @@ class UniqueFileTest(test_util.TempDirTestCase): """Tests for certbot.util.unique_file.""" def setUp(self): - super(UniqueFileTest, self).setUp() + super().setUp() self.default_name = os.path.join(self.tempdir, "foo.txt") @@ -260,7 +260,7 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): def test_basic(self): f, path = self._call("wow") - self.assertTrue(isinstance(f, file_type)) + self.assertIsInstance(f, file_type) self.assertEqual(os.path.join(self.tempdir, "wow.conf"), path) f.close() @@ -269,9 +269,9 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): for _ in range(10): items.append(self._call("wow")) f, name = items[-1] - self.assertTrue(isinstance(f, file_type)) - self.assertTrue(isinstance(name, str)) - self.assertTrue("wow-0009.conf" in name) + self.assertIsInstance(f, file_type) + self.assertIsInstance(name, str) + self.assertIn("wow-0009.conf", name) for f, _ in items: f.close() @@ -284,7 +284,7 @@ class SafelyRemoveTest(test_util.TempDirTestCase): """Tests for certbot.util.safely_remove.""" def setUp(self): - super(SafelyRemoveTest, self).setUp() + super().setUp() self.path = os.path.join(self.tempdir, "foo") @@ -349,7 +349,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): with mock.patch("certbot.util.logger.warning") as mock_warn: self.parser.parse_args(["--old-option"]) self.assertEqual(mock_warn.call_count, 1) - self.assertTrue("is deprecated" in mock_warn.call_args[0][0]) + self.assertIn("is deprecated", mock_warn.call_args[0][0]) self.assertEqual("--old-option", mock_warn.call_args[0][1]) def test_warning_with_arg(self): @@ -357,7 +357,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): with mock.patch("certbot.util.logger.warning") as mock_warn: self.parser.parse_args(["--old-option", "42"]) self.assertEqual(mock_warn.call_count, 1) - self.assertTrue("is deprecated" in mock_warn.call_args[0][0]) + self.assertIn("is deprecated", mock_warn.call_args[0][0]) self.assertEqual("--old-option", mock_warn.call_args[0][1]) def test_help(self): @@ -368,7 +368,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): self.parser.parse_args(["-h"]) except SystemExit: pass - self.assertTrue("--old-option" not in stdout.getvalue()) + self.assertNotIn("--old-option", stdout.getvalue()) def test_set_constant(self): """Test when ACTION_TYPES_THAT_DONT_NEED_A_VALUE is a set. @@ -519,7 +519,7 @@ class OsInfoTest(unittest.TestCase): m_distro.like.return_value = "first debian third" id_likes = cbutil.get_systemd_os_like() self.assertEqual(len(id_likes), 3) - self.assertTrue("debian" in id_likes) + self.assertIn("debian", id_likes) @mock.patch("certbot.util.distro") @unittest.skipUnless(sys.platform.startswith("linux"), "requires Linux") @@ -610,7 +610,7 @@ class AtexitRegisterTest(unittest.TestCase): def test_not_called(self): self._test_common(initial_pid=-1) - self.assertFalse(self.func.called) + self.assertIs(self.func.called, False) def _test_common(self, initial_pid): with mock.patch('certbot.util._INITIAL_PID', initial_pid): diff --git a/letsencrypt-auto b/letsencrypt-auto index 002fd5ffc..c37c45596 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.12.0" +LE_AUTO_VERSION="1.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -800,12 +800,14 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -1487,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.14.0 \ + --hash=sha256:67b4d26ceaea6c7f8325d0d45169e7a165a2cabc7122c84bc971ba068ca19cca \ + --hash=sha256:959ea90c6bb8dca38eab9772722cb940972ef6afcd5f15deef08b3c3636841eb +acme==1.14.0 \ + --hash=sha256:4f48c41261202f1a389ec2986b2580b58f53e0d5a1ae2463b34318d78b87fc66 \ + --hash=sha256:61daccfb0343628cbbca551a7fc4c82482113952c21db3fe0c585b7c98fa1c35 +certbot-apache==1.14.0 \ + --hash=sha256:b757038db23db707c44630fecb46e99172bd791f0db5a8e623c0842613c4d3d9 \ + --hash=sha256:887fe4a21af2de1e5c2c9428bacba6eb7c1219257bc70f1a1d8447c8a321adb0 +certbot-nginx==1.14.0 \ + --hash=sha256:8916a815437988d6c192df9f035bb7a176eab20eee0956677b335d0698d243fb \ + --hash=sha256:cc2a8a0de56d9bb6b2efbda6c80c647dad8db2bb90675cac03ade94bd5fc8597 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/build.py b/letsencrypt-auto-source/build.py deleted file mode 100755 index a1e40fe44..000000000 --- a/letsencrypt-auto-source/build.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -"""Stitch together the letsencrypt-auto script. - -Implement a simple templating language in which {{ some/file }} turns into the -contents of the file at ./pieces/some/file except for certain tokens which have -other, special definitions. - -""" -from os.path import abspath, dirname, join -import re - -from version import certbot_version, file_contents - - -DIR = dirname(abspath(__file__)) - - -def build(version=None, requirements=None): - """Return the built contents of the letsencrypt-auto script. - - :arg version: The version to attach to the script. Default: the version of - the certbot package - :arg requirements: The contents of the requirements file to embed. Default: - contents of dependency-requirements.txt, letsencrypt-requirements.txt, - and certbot-requirements.txt - - """ - special_replacements = { - 'LE_AUTO_VERSION': version or certbot_version(DIR) - } - if requirements: - special_replacements['dependency-requirements.txt'] = '' - special_replacements['letsencrypt-requirements.txt'] = '' - special_replacements['certbot-requirements.txt'] = requirements - - def replacer(match): - token = match.group(1) - if token in special_replacements: - return special_replacements[token] - else: - return file_contents(join(DIR, 'pieces', token)) - - return re.sub(r'{{\s*([A-Za-z0-9_./-]+)\s*}}', - replacer, - file_contents(join(DIR, 'letsencrypt-auto.template'))) - - -def main(): - with open(join(DIR, 'letsencrypt-auto'), 'w') as out: - out.write(build()) - - -if __name__ == '__main__': - main() diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index aba5f1140..c0cf63418 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmAZorcACgkQTRfJlc2X -dfI6Ogf+LFASyH9sgTV1k9hs1zbmO3CxyE9QQs1JLXpoKOQ1tKv+v+kpt+lJ005g -rielyRSssXtZSyfLchCSBh6qaEBodoOcz8RS2z7rDnR9jKOJv252Buh2oSa3KPmn -WPjRmB3zVXnhq/XmPKQTnoflUlBg+MtZuZXt0Fvu8rvQB+RY3AUfB5Xs83nxJNj4 -W9qNpZYl0sJWWiydr23bEk35MJSt62sKDvyqIVjUfgDfXHmauOpg0foz2xS6XP8i -Ke66GUKaQ1ap2BTucwVT0hieXiQZpxx1PitUeEOjOH9PUfrAxyFlQ0XQaVlqoBhc -YM3nzJw9yf12b+XCUvMzHyQmDA5vdQ== -=AUGt +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmBsmUkACgkQTRfJlc2X +dfI7Bwf9FkNrf1HEh2G3uk1p+qLMd/s5kcVV2udK2FkRELee5nHlLZx2YmHA/8ID +gqsk8EsyRZNMX374nGrPm0syykdEsyVtMJTbHCEr+Ms3l54ZgE3HV6ywnhWSlAFo +Za50kdzhodBVTS5AEADbCKLKObVAWwO3fFKtKyv/iY29ykpHK0KSHCKRII3iQU7l +dnR6u35Z0wgfEmDxsH27K6uo0YepZaEL70qHHFk93MhCh9Z15rO17gRpsVzz7Z1j +YClI6h2K/VOfZtbkoQvoks7s+xd75Kjr3GNH+cznkJx8gNWSZLfkc1XX4Bjdm4GG +IWz3Ezy8tFg6PtITb7y+aIg75kWx4w== +=zEy4 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 224abaf01..c37c45596 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.13.0.dev0" +LE_AUTO_VERSION="1.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -800,6 +800,7 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 @@ -1488,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.14.0 \ + --hash=sha256:67b4d26ceaea6c7f8325d0d45169e7a165a2cabc7122c84bc971ba068ca19cca \ + --hash=sha256:959ea90c6bb8dca38eab9772722cb940972ef6afcd5f15deef08b3c3636841eb +acme==1.14.0 \ + --hash=sha256:4f48c41261202f1a389ec2986b2580b58f53e0d5a1ae2463b34318d78b87fc66 \ + --hash=sha256:61daccfb0343628cbbca551a7fc4c82482113952c21db3fe0c585b7c98fa1c35 +certbot-apache==1.14.0 \ + --hash=sha256:b757038db23db707c44630fecb46e99172bd791f0db5a8e623c0842613c4d3d9 \ + --hash=sha256:887fe4a21af2de1e5c2c9428bacba6eb7c1219257bc70f1a1d8447c8a321adb0 +certbot-nginx==1.14.0 \ + --hash=sha256:8916a815437988d6c192df9f035bb7a176eab20eee0956677b335d0698d243fb \ + --hash=sha256:cc2a8a0de56d9bb6b2efbda6c80c647dad8db2bb90675cac03ade94bd5fc8597 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index ac143de51..b297bdfb3 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template deleted file mode 100755 index 70b75176e..000000000 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ /dev/null @@ -1,785 +0,0 @@ -#!/bin/sh -# -# Download and run the latest release version of the Certbot client. -# -# NOTE: THIS SCRIPT IS AUTO-GENERATED AND SELF-UPDATING -# -# IF YOU WANT TO EDIT IT LOCALLY, *ALWAYS* RUN YOUR COPY WITH THE -# "--no-self-upgrade" FLAG -# -# IF YOU WANT TO SEND PULL REQUESTS, THE REAL SOURCE FOR THIS FILE IS -# letsencrypt-auto-source/letsencrypt-auto.template AND -# letsencrypt-auto-source/pieces/bootstrappers/* - -set -e # Work even if somebody does "sh thisscript.sh". - -# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, -# if you want to change where the virtual environment will be installed - -# HOME might not be defined when being run through something like systemd -if [ -z "$HOME" ]; then - HOME=~root -fi -if [ -z "$XDG_DATA_HOME" ]; then - XDG_DATA_HOME=~/.local/share -fi -if [ -z "$VENV_PATH" ]; then - # We export these values so they are preserved properly if this script is - # rerun with sudo/su where $HOME/$XDG_DATA_HOME may have a different value. - export OLD_VENV_PATH="$XDG_DATA_HOME/letsencrypt" - export VENV_PATH="/opt/eff.org/certbot/venv" -fi -VENV_BIN="$VENV_PATH/bin" -BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="{{ LE_AUTO_VERSION }}" -BASENAME=$(basename $0) -USAGE="Usage: $BASENAME [OPTIONS] -A self-updating wrapper script for the Certbot ACME client. When run, updates -to both this script and certbot will be downloaded and installed. After -ensuring you have the latest versions installed, certbot will be invoked with -all arguments you have provided. - -Help for certbot itself cannot be provided until it is installed. - - --debug attempt experimental installation - -h, --help print this help - -n, --non-interactive, --noninteractive run without asking for user input - --no-bootstrap do not install OS dependencies - --no-permissions-check do not warn about file system permissions - --no-self-upgrade do not download updates - --os-packages-only install OS dependencies and exit - --install-only install certbot, upgrade if needed, and exit - -v, --verbose provide more output - -q, --quiet provide only update/error output; - implies --non-interactive - -All arguments are accepted and forwarded to the Certbot client when run." -export CERTBOT_AUTO="$0" - -for arg in "$@" ; do - case "$arg" in - --debug) - DEBUG=1;; - --os-packages-only) - OS_PACKAGES_ONLY=1;; - --install-only) - INSTALL_ONLY=1;; - --no-self-upgrade) - # Do not upgrade this script (also prevents client upgrades, because each - # copy of the script pins a hash of the python client) - NO_SELF_UPGRADE=1;; - --no-permissions-check) - NO_PERMISSIONS_CHECK=1;; - --no-bootstrap) - NO_BOOTSTRAP=1;; - --help) - HELP=1;; - --noninteractive|--non-interactive) - NONINTERACTIVE=1;; - --quiet) - QUIET=1;; - renew) - ASSUME_YES=1;; - --verbose) - VERBOSE=1;; - -[!-]*) - OPTIND=1 - while getopts ":hnvq" short_arg $arg; do - case "$short_arg" in - h) - HELP=1;; - n) - NONINTERACTIVE=1;; - q) - QUIET=1;; - v) - VERBOSE=1;; - esac - done;; - esac -done - -if [ $BASENAME = "letsencrypt-auto" ]; then - # letsencrypt-auto does not respect --help or --yes for backwards compatibility - NONINTERACTIVE=1 - HELP=0 -fi - -# Set ASSUME_YES to 1 if QUIET or NONINTERACTIVE -if [ "$QUIET" = 1 -o "$NONINTERACTIVE" = 1 ]; then - ASSUME_YES=1 -fi - -say() { - if [ "$QUIET" != 1 ]; then - echo "$@" - fi -} - -error() { - echo "$@" -} - -# Support for busybox and others where there is no "command", -# but "which" instead -if command -v command > /dev/null 2>&1 ; then - export EXISTS="command -v" -elif which which > /dev/null 2>&1 ; then - export EXISTS="which" -else - error "Cannot find command nor which... please install one!" - exit 1 -fi - -# Certbot itself needs root access for almost all modes of operation. -# certbot-auto needs root access to bootstrap OS dependencies and install -# Certbot at a protected path so it can be safely run as root. To accomplish -# this, this script will attempt to run itself as root if it doesn't have the -# necessary privileges by using `sudo` or falling back to `su` if it is not -# available. The mechanism used to obtain root access can be set explicitly by -# setting the environment variable LE_AUTO_SUDO to 'sudo', 'su', 'su_sudo', -# 'SuSudo', or '' as used below. - -# Because the parameters in `su -c` has to be a string, -# we need to properly escape it. -SuSudo() { - args="" - # This `while` loop iterates over all parameters given to this function. - # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrapped in a pair of `'`, then appended to `$args` string - # For example, `echo "It's only 1\$\!"` will be escaped to: - # 'echo' 'It'"'"'s only 1$!' - # │ │└┼┘│ - # │ │ │ └── `'s only 1$!'` the literal string - # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings following it - # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself - while [ $# -ne 0 ]; do - args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " - shift - done - su root -c "$args" -} - -# Sets the environment variable SUDO to be the name of the program or function -# to call to get root access. If this script already has root privleges, SUDO -# is set to an empty string. The value in SUDO should be run with the command -# to called with root privileges as arguments. -SetRootAuthMechanism() { - SUDO="" - if [ -n "${LE_AUTO_SUDO+x}" ]; then - case "$LE_AUTO_SUDO" in - SuSudo|su_sudo|su) - SUDO=SuSudo - ;; - sudo) - SUDO="sudo -E" - ;; - '') - # If we're not running with root, don't check that this script can only - # be modified by system users and groups. - NO_PERMISSIONS_CHECK=1 - ;; - *) - error "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." - exit 1 - esac - say "Using preset root authorization mechanism '$LE_AUTO_SUDO'." - else - if test "`id -u`" -ne "0" ; then - if $EXISTS sudo 1>/dev/null 2>&1; then - SUDO="sudo -E" - else - say \"sudo\" is not available, will use \"su\" for installation steps... - SUDO=SuSudo - fi - fi - fi -} - -if [ "$1" = "--cb-auto-has-root" ]; then - shift 1 -else - SetRootAuthMechanism - if [ -n "$SUDO" ]; then - say "Requesting to rerun $0 with root privileges..." - $SUDO "$0" --cb-auto-has-root "$@" - exit 0 - fi -fi - -# Runs this script again with the given arguments. --cb-auto-has-root is added -# to the command line arguments to ensure we don't try to acquire root a -# second time. After the script is rerun, we exit the current script. -RerunWithArgs() { - "$0" --cb-auto-has-root "$@" - exit 0 -} - -BootstrapMessage() { - # Arguments: Platform name - say "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" -} - -ExperimentalBootstrap() { - # Arguments: Platform name, bootstrap function name - if [ "$DEBUG" = 1 ]; then - if [ "$2" != "" ]; then - BootstrapMessage $1 - $2 - fi - else - error "FATAL: $1 support is very experimental at present..." - error "if you would like to work on improving it, please ensure you have backups" - error "and then run this script again with the --debug flag!" - error "Alternatively, you can install OS dependencies yourself and run this script" - error "again with --no-bootstrap." - exit 1 - fi -} - -DeprecationBootstrap() { - # Arguments: Platform name, bootstrap function name - if [ "$DEBUG" = 1 ]; then - if [ "$2" != "" ]; then - BootstrapMessage $1 - $2 - fi - else - error "WARNING: certbot-auto support for this $1 is DEPRECATED!" - error "Please visit certbot.eff.org to learn how to download a version of" - error "Certbot that is packaged for your system. While an existing version" - error "of certbot-auto may work currently, we have stopped supporting updating" - error "system packages for your system. Please switch to a packaged version" - error "as soon as possible." - exit 1 - fi -} - -MIN_PYTHON_2_VERSION="2.7" -MIN_PYVER2=$(echo "$MIN_PYTHON_2_VERSION" | sed 's/\.//') -MIN_PYTHON_3_VERSION="3.6" -MIN_PYVER3=$(echo "$MIN_PYTHON_3_VERSION" | sed 's/\.//') -# Sets LE_PYTHON to Python version string and PYVER to the first two -# digits of the python version. -# MIN_PYVER and MIN_PYTHON_VERSION are also set by this function, and their -# values depend on if we try to use Python 3 or Python 2. -DeterminePythonVersion() { - # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python - # - # If no Python is found, PYVER is set to 0. - if [ "$USE_PYTHON_3" = 1 ]; then - MIN_PYVER=$MIN_PYVER3 - MIN_PYTHON_VERSION=$MIN_PYTHON_3_VERSION - for LE_PYTHON in "$LE_PYTHON" python3; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - else - MIN_PYVER=$MIN_PYVER2 - MIN_PYTHON_VERSION=$MIN_PYTHON_2_VERSION - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - fi - if [ "$?" != "0" ]; then - if [ "$1" != "NOCRASH" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 - else - PYVER=0 - return 0 - fi - fi - - PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') - if [ "$PYVER" -lt "$MIN_PYVER" ]; then - if [ "$1" != "NOCRASH" ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." - exit 1 - fi - fi -} - -{{ bootstrappers/deb_common.sh }} -{{ bootstrappers/rpm_common_base.sh }} -{{ bootstrappers/rpm_common.sh }} -{{ bootstrappers/rpm_python3_legacy.sh }} -{{ bootstrappers/rpm_python3.sh }} -{{ bootstrappers/suse_common.sh }} -{{ bootstrappers/arch_common.sh }} -{{ bootstrappers/gentoo_common.sh }} -{{ bootstrappers/free_bsd.sh }} -{{ bootstrappers/mac.sh }} -{{ bootstrappers/smartos.sh }} -{{ bootstrappers/mageia_common.sh }} - -# Set Bootstrap to the function that installs OS dependencies on this system -# and BOOTSTRAP_VERSION to the unique identifier for the current version of -# that function. If Bootstrap is set to a function that doesn't install any -# packages BOOTSTRAP_VERSION is not set. -if [ -f /etc/debian_version ]; then - DEPRECATED_OS=1 -elif [ -f /etc/mageia-release ]; then - # Mageia has both /etc/mageia-release and /etc/redhat-release - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/redhat-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 - # Run DeterminePythonVersion to decide on the basis of available Python versions - # whether to use 2.x or 3.x on RedHat-like systems. - # Then, revert LE_PYTHON to its previous state. - prev_le_python="$LE_PYTHON" - unset LE_PYTHON - DeterminePythonVersion "NOCRASH" - - RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"` - - if [ "$PYVER" -eq 26 -a $(uname -m) != 'x86_64' ]; then - # 32 bits CentOS 6 and affiliates are not supported anymore by certbot-auto. - DEPRECATED_OS=1 - fi - - # Set RPM_DIST_VERSION to VERSION_ID from /etc/os-release after splitting on - # '.' characters (e.g. "8.0" becomes "8"). If the command exits with an - # error, RPM_DIST_VERSION is set to "unknown". - RPM_DIST_VERSION=$( (. /etc/os-release 2> /dev/null && echo "$VERSION_ID") | cut -d '.' -f1 || echo "unknown") - - # If RPM_DIST_VERSION is an empty string or it contains any nonnumeric - # characters, the value is unexpected so we set RPM_DIST_VERSION to 0. - if [ -z "$RPM_DIST_VERSION" ] || [ -n "$(echo "$RPM_DIST_VERSION" | tr -d '[0-9]')" ]; then - RPM_DIST_VERSION=0 - fi - - # Handle legacy RPM distributions - if [ "$PYVER" -eq 26 ]; then - # Check if an automated bootstrap can be achieved on this system. - if ! Python36SclIsAvailable; then - INTERACTIVE_BOOTSTRAP=1 - fi - - USE_PYTHON_3=1 - - # Try now to enable SCL rh-python36 for systems already bootstrapped - # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto - EnablePython36SCL - else - # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. - # RHEL 8 also uses python3 by default. - if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 ]; then - RPM_USE_PYTHON_3=1 - elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then - RPM_USE_PYTHON_3=1 - elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then - RPM_USE_PYTHON_3=1 - else - RPM_USE_PYTHON_3=0 - fi - - if [ "$RPM_USE_PYTHON_3" = 1 ]; then - USE_PYTHON_3=1 - fi - fi - - LE_PYTHON="$prev_le_python" -elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/arch-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/manjaro-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/gentoo-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif uname | grep -iq FreeBSD ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif uname | grep -iq Darwin ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -else - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -fi - -# We handle this case after determining the normal bootstrap version to allow -# variables like USE_PYTHON_3 to be properly set. As described above, if the -# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not -# be set so we unset it here. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } - unset BOOTSTRAP_VERSION -fi - -if [ "$DEPRECATED_OS" = 1 ]; then - Bootstrap() { - error "Skipping bootstrap because certbot-auto is deprecated on this system." - } - unset BOOTSTRAP_VERSION -fi - -# Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used -# to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set -# if it is unknown how OS dependencies were installed on this system. -SetPrevBootstrapVersion() { - if [ -f $BOOTSTRAP_VERSION_PATH ]; then - PREV_BOOTSTRAP_VERSION=$(cat "$BOOTSTRAP_VERSION_PATH") - # The list below only contains bootstrap version strings that existed before - # we started writing them to disk. - # - # DO NOT MODIFY THIS LIST UNLESS YOU KNOW WHAT YOU'RE DOING! - elif grep -Fqx "$BOOTSTRAP_VERSION" << "UNLIKELY_EOF" -BootstrapDebCommon 1 -BootstrapMageiaCommon 1 -BootstrapRpmCommon 1 -BootstrapSuseCommon 1 -BootstrapArchCommon 1 -BootstrapGentooCommon 1 -BootstrapFreeBsd 1 -BootstrapMac 1 -BootstrapSmartOS 1 -UNLIKELY_EOF - then - # If there's no bootstrap version saved to disk, but the currently selected - # bootstrap script is from before we started saving the version number, - # return the currently selected version to prevent us from rebootstrapping - # unnecessarily. - PREV_BOOTSTRAP_VERSION="$BOOTSTRAP_VERSION" - fi -} - -TempDir() { - mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS -} - -# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, -# returns a non-zero number. -OldVenvExists() { - [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] -} - -# Given python path, version 1 and version 2, check if version 1 is outdated compared to version 2. -# An unofficial version provided as version 1 (eg. 0.28.0.dev0) will be treated -# specifically by printing "UNOFFICIAL". Otherwise, print "OUTDATED" if version 1 -# is outdated, and "UP_TO_DATE" if not. -# This function relies only on installed python environment (2.x or 3.x) by certbot-auto. -CompareVersions() { - "$1" - "$2" "$3" << "UNLIKELY_EOF" -import sys -from distutils.version import StrictVersion - -try: - current = StrictVersion(sys.argv[1]) -except ValueError: - sys.stdout.write('UNOFFICIAL') - sys.exit() - -try: - remote = StrictVersion(sys.argv[2]) -except ValueError: - sys.stdout.write('UP_TO_DATE') - sys.exit() - -if current < remote: - sys.stdout.write('OUTDATED') -else: - sys.stdout.write('UP_TO_DATE') -UNLIKELY_EOF -} - -# Create a new virtual environment for Certbot. It will overwrite any existing one. -# Parameters: LE_PYTHON, VENV_PATH, PYVER, VERBOSE -CreateVenv() { - "$1" - "$2" "$3" "$4" << "UNLIKELY_EOF" -{{ create_venv.py }} -UNLIKELY_EOF -} - -# Check that the given PATH_TO_CHECK has secured permissions. -# Parameters: LE_PYTHON, PATH_TO_CHECK -CheckPathPermissions() { - "$1" - "$2" << "UNLIKELY_EOF" -{{ check_permissions.py }} -UNLIKELY_EOF -} - -if [ "$1" = "--le-auto-phase2" ]; then - # Phase 2: Create venv, install LE, and run. - - shift 1 # the --le-auto-phase2 arg - - if [ "$DEPRECATED_OS" = 1 ]; then - # Phase 2 damage control mode for deprecated OSes. - # In this situation, we bypass any bootstrap or certbot venv setup. - error "Your system is not supported by certbot-auto anymore." - - if [ ! -d "$VENV_PATH" ] && OldVenvExists; then - VENV_BIN="$OLD_VENV_PATH/bin" - fi - - if [ -f "$VENV_BIN/letsencrypt" -a "$INSTALL_ONLY" != 1 ]; then - error "certbot-auto and its Certbot installation will no longer receive updates." - error "You will not receive any bug fixes including those fixing server compatibility" - error "or security problems." - error "Please visit https://certbot.eff.org/ to check for other alternatives." - "$VENV_BIN/letsencrypt" "$@" - exit 0 - else - error "Certbot cannot be installed." - error "Please visit https://certbot.eff.org/ to check for other alternatives." - exit 1 - fi - fi - - SetPrevBootstrapVersion - - if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then - unset LE_PYTHON - fi - - INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ] || OldVenvExists; then - # If the selected Bootstrap function isn't a noop and it differs from the - # previously used version - if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then - # Check if we can rebootstrap without manual user intervention: this requires that - # certbot-auto is in non-interactive mode AND selected bootstrap does not claim to - # require a manual user intervention. - if [ "$NONINTERACTIVE" = 1 -a "$INTERACTIVE_BOOTSTRAP" != 1 ]; then - CAN_REBOOTSTRAP=1 - fi - # Check if rebootstrap can be done non-interactively and current shell is non-interactive - # (true if stdin and stdout are not attached to a terminal). - if [ \( "$CAN_REBOOTSTRAP" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - if [ -d "$VENV_PATH" ]; then - rm -rf "$VENV_PATH" - fi - # In the case the old venv was just a symlink to the new one, - # OldVenvExists is now false because we deleted the venv at VENV_PATH. - if OldVenvExists; then - rm -rf "$OLD_VENV_PATH" - ln -s "$VENV_PATH" "$OLD_VENV_PATH" - fi - RerunWithArgs "$@" - # Otherwise bootstrap needs to be done manually by the user. - else - # If it is because bootstrapping is interactive, --non-interactive will be of no use. - if [ "$INTERACTIVE_BOOTSTRAP" = 1 ]; then - error "Skipping upgrade because new OS dependencies may need to be installed." - error "This requires manual user intervention: please run this script again manually." - # If this is because of the environment (eg. non interactive shell without - # --non-interactive flag set), help the user in that direction. - else - error "Skipping upgrade because new OS dependencies may need to be installed." - error - error "To upgrade to a newer version, please run this script again manually so you can" - error "approve changes or with --non-interactive on the command line to automatically" - error "install any required packages." - fi - # Set INSTALLED_VERSION to be the same so we don't update the venv - INSTALLED_VERSION="$LE_AUTO_VERSION" - # Continue to use OLD_VENV_PATH if the new venv doesn't exist - if [ ! -d "$VENV_PATH" ]; then - VENV_BIN="$OLD_VENV_PATH/bin" - fi - fi - elif [ -f "$VENV_BIN/letsencrypt" ]; then - # --version output ran through grep due to python-cryptography DeprecationWarnings - # grep for both certbot and letsencrypt until certbot and shim packages have been released - INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) - if [ -z "$INSTALLED_VERSION" ]; then - error "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 - "$VENV_BIN/letsencrypt" --version - exit 1 - fi - fi - fi - - if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then - say "Creating virtual environment..." - DeterminePythonVersion - CreateVenv "$LE_PYTHON" "$VENV_PATH" "$PYVER" "$VERBOSE" - - if [ -n "$BOOTSTRAP_VERSION" ]; then - echo "$BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH" - elif [ -n "$PREV_BOOTSTRAP_VERSION" ]; then - echo "$PREV_BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH" - fi - - say "Installing Python packages..." - TEMP_DIR=$(TempDir) - trap 'rm -rf "$TEMP_DIR"' EXIT - # There is no $ interpolation due to quotes on starting heredoc delimiter. - # ------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" -{{ dependency-requirements.txt }} -{{ letsencrypt-requirements.txt }} -{{ certbot-requirements.txt }} -UNLIKELY_EOF - # ------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" -{{ pipstrap.py }} -UNLIKELY_EOF - # ------------------------------------------------------------------------- - # Set PATH so pipstrap upgrades the right (v)env: - PATH="$VENV_BIN:$PATH" "$VENV_BIN/python" "$TEMP_DIR/pipstrap.py" - set +e - if [ "$VERBOSE" = 1 ]; then - "$VENV_BIN/pip" install --disable-pip-version-check --no-cache-dir --require-hashes -r "$TEMP_DIR/letsencrypt-auto-requirements.txt" - else - PIP_OUT=`"$VENV_BIN/pip" install --disable-pip-version-check --no-cache-dir --require-hashes -r "$TEMP_DIR/letsencrypt-auto-requirements.txt" 2>&1` - fi - PIP_STATUS=$? - set -e - if [ "$PIP_STATUS" != 0 ]; then - # Report error. (Otherwise, be quiet.) - error "Had a problem while installing Python packages." - if [ "$VERBOSE" != 1 ]; then - error - error "pip prints the following errors: " - error "=====================================================" - error "$PIP_OUT" - error "=====================================================" - error - error "Certbot has problem setting up the virtual environment." - - if `echo $PIP_OUT | grep -q Killed` || `echo $PIP_OUT | grep -q "allocate memory"` ; then - error - error "Based on your pip output, the problem can likely be fixed by " - error "increasing the available memory." - else - error - error "We were not be able to guess the right solution from your pip " - error "output." - fi - - error - error "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" - error "for possible solutions." - error "You may also find some support resources at https://certbot.eff.org/support/ ." - fi - rm -rf "$VENV_PATH" - exit 1 - fi - - if [ -d "$OLD_VENV_PATH" -a ! -L "$OLD_VENV_PATH" ]; then - rm -rf "$OLD_VENV_PATH" - ln -s "$VENV_PATH" "$OLD_VENV_PATH" - fi - - say "Installation succeeded." - fi - - # If you're modifying any of the code after this point in this current `if` block, you - # may need to update the "$DEPRECATED_OS" = 1 case at the beginning of phase 2 as well. - - if [ "$INSTALL_ONLY" = 1 ]; then - say "Certbot is installed." - exit 0 - fi - - "$VENV_BIN/letsencrypt" "$@" - -else - # Phase 1: Upgrade certbot-auto if necessary, then self-invoke. - # - # Each phase checks the version of only the thing it is responsible for - # upgrading. Phase 1 checks the version of the latest release of - # certbot-auto (which is always the same as that of the certbot - # package). Phase 2 checks the version of the locally installed certbot. - export PHASE_1_VERSION="$LE_AUTO_VERSION" - - if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if ! OldVenvExists; then - if [ "$HELP" = 1 ]; then - echo "$USAGE" - exit 0 - fi - # If it looks like we've never bootstrapped before, bootstrap: - Bootstrap - fi - fi - if [ "$OS_PACKAGES_ONLY" = 1 ]; then - say "OS packages installed." - exit 0 - fi - - DeterminePythonVersion "NOCRASH" - # Don't warn about file permissions if the user disabled the check or we - # can't find an up-to-date Python. - if [ "$PYVER" -ge "$MIN_PYVER" -a "$NO_PERMISSIONS_CHECK" != 1 ]; then - # If the script fails for some reason, don't break certbot-auto. - set +e - # Suppress unexpected error output. - CHECK_PERM_OUT=$(CheckPathPermissions "$LE_PYTHON" "$0" 2>/dev/null) - CHECK_PERM_STATUS="$?" - set -e - # Only print output if the script ran successfully and it actually produced - # output. The latter check resolves - # https://github.com/certbot/certbot/issues/7012. - if [ "$CHECK_PERM_STATUS" = 0 -a -n "$CHECK_PERM_OUT" ]; then - error "$CHECK_PERM_OUT" - fi - fi - - if [ "$NO_SELF_UPGRADE" != 1 ]; then - TEMP_DIR=$(TempDir) - trap 'rm -rf "$TEMP_DIR"' EXIT - # --------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" -{{ fetch.py }} -UNLIKELY_EOF - # --------------------------------------------------------------------------- - if [ "$PYVER" -lt "$MIN_PYVER" ]; then - error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." - elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then - error "WARNING: unable to check for updates." - fi - - # If for any reason REMOTE_VERSION is not set, let's assume certbot-auto is up-to-date, - # and do not go into the self-upgrading process. - if [ -n "$REMOTE_VERSION" ]; then - LE_VERSION_STATE=`CompareVersions "$LE_PYTHON" "$LE_AUTO_VERSION" "$REMOTE_VERSION"` - - if [ "$LE_VERSION_STATE" = "UNOFFICIAL" ]; then - say "Unofficial certbot-auto version detected, self-upgrade is disabled: $LE_AUTO_VERSION" - elif [ "$LE_VERSION_STATE" = "OUTDATED" ]; then - say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." - - # Now we drop into Python so we don't have to install even more - # dependencies (curl, etc.), for better flow control, and for the option of - # future Windows compatibility. - "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" - - # Install new copy of certbot-auto. - # TODO: Deal with quotes in pathnames. - say "Replacing certbot-auto..." - # Clone permissions with cp. chmod and chown don't have a --reference - # option on macOS or BSD, and stat -c on Linux is stat -f on macOS and BSD: - cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" - cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" - # Using mv rather than cp leaves the old file descriptor pointing to the - # original copy so the shell can continue to read it unmolested. mv across - # filesystems is non-atomic, doing `rm dest, cp src dest, rm src`, but the - # cp is unlikely to fail if the rm doesn't. - mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" - fi # A newer version is available. - fi - fi # Self-upgrading is allowed. - - RerunWithArgs --le-auto-phase2 "$@" -fi diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh deleted file mode 100755 index 3be78d3f8..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ /dev/null @@ -1,37 +0,0 @@ -# If new packages are installed by BootstrapArchCommon below, this version -# number must be increased. -BOOTSTRAP_ARCH_COMMON_VERSION=1 - -BootstrapArchCommon() { - # Tested with: - # - ArchLinux (x86_64) - # - # "python-virtualenv" is Python3, but "python2-virtualenv" provides - # only "virtualenv2" binary, not "virtualenv". - - deps=" - python2 - python-virtualenv - gcc - augeas - openssl - libffi - ca-certificates - pkg-config - " - - # pacman -T exits with 127 if there are missing dependencies - missing=$(pacman -T $deps) || true - - if [ "$ASSUME_YES" = 1 ]; then - noconfirm="--noconfirm" - fi - - if [ "$missing" ]; then - if [ "$QUIET" = 1 ]; then - pacman -S --needed $missing $noconfirm > /dev/null - else - pacman -S --needed $missing $noconfirm - fi - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh deleted file mode 100644 index 93bdc63b4..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ /dev/null @@ -1,67 +0,0 @@ -# If new packages are installed by BootstrapDebCommon below, this version -# number must be increased. -BOOTSTRAP_DEB_COMMON_VERSION=1 - -BootstrapDebCommon() { - # Current version tested with: - # - # - Ubuntu - # - 14.04 (x64) - # - 15.04 (x64) - # - Debian - # - 7.9 "wheezy" (x64) - # - sid (2015-10-21) (x64) - - # Past versions tested with: - # - # - Debian 8.0 "jessie" (x64) - # - Raspbian 7.8 (armhf) - - # Believed not to work: - # - # - Debian 6.0.10 "squeeze" (x64) - - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='-qq' - fi - - apt-get $QUIET_FLAG update || error apt-get update hit problems but continuing anyway... - - # virtualenv binary can be found in different packages depending on - # distro version (#346) - - virtualenv= - # virtual env is known to apt and is installable - if apt-cache show virtualenv > /dev/null 2>&1 ; then - if ! LC_ALL=C apt-cache --quiet=0 show virtualenv 2>&1 | grep -q 'No packages found'; then - virtualenv="virtualenv" - fi - fi - - if apt-cache show python-virtualenv > /dev/null 2>&1; then - virtualenv="$virtualenv python-virtualenv" - fi - - augeas_pkg="libaugeas0 augeas-lenses" - - if [ "$ASSUME_YES" = 1 ]; then - YES_FLAG="-y" - fi - - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ - python \ - python-dev \ - $virtualenv \ - gcc \ - $augeas_pkg \ - libssl-dev \ - openssl \ - libffi-dev \ - ca-certificates \ - - - if ! $EXISTS virtualenv > /dev/null ; then - error Failed to install a working \"virtualenv\" command, exiting - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh b/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh deleted file mode 100755 index a67c85619..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh +++ /dev/null @@ -1,15 +0,0 @@ -# If new packages are installed by BootstrapFreeBsd below, this version number -# must be increased. -BOOTSTRAP_FREEBSD_VERSION=1 - -BootstrapFreeBsd() { - if [ "$QUIET" = 1 ]; then - QUIET_FLAG="--quiet" - fi - - pkg install -Ay $QUIET_FLAG \ - python \ - py27-virtualenv \ - augeas \ - libffi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh deleted file mode 100755 index e2d24b5fb..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh +++ /dev/null @@ -1,31 +0,0 @@ -# If new packages are installed by BootstrapGentooCommon below, this version -# number must be increased. -BOOTSTRAP_GENTOO_COMMON_VERSION=1 - -BootstrapGentooCommon() { - PACKAGES=" - dev-lang/python:2.7 - dev-python/virtualenv - app-admin/augeas - dev-libs/openssl - dev-libs/libffi - app-misc/ca-certificates - virtual/pkgconfig" - - ASK_OPTION="--ask" - if [ "$ASSUME_YES" = 1 ]; then - ASK_OPTION="" - fi - - case "$PACKAGE_MANAGER" in - (paludis) - cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x - ;; - (pkgcore) - pmerge --noreplace --oneshot $ASK_OPTION $PACKAGES - ;; - (portage|*) - emerge --noreplace --oneshot $ASK_OPTION $PACKAGES - ;; - esac -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh deleted file mode 100755 index 9e26d3389..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh +++ /dev/null @@ -1,48 +0,0 @@ -# If new packages are installed by BootstrapMac below, this version number must -# be increased. -BOOTSTRAP_MAC_VERSION=1 - -BootstrapMac() { - if hash brew 2>/dev/null; then - say "Using Homebrew to install dependencies..." - pkgman=brew - pkgcmd="brew install" - elif hash port 2>/dev/null; then - say "Using MacPorts to install dependencies..." - pkgman=port - pkgcmd="port install" - else - say "No Homebrew/MacPorts; installing Homebrew..." - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - pkgman=brew - pkgcmd="brew install" - fi - - $pkgcmd augeas - if [ "$(which python)" = "/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python" \ - -o "$(which python)" = "/usr/bin/python" ]; then - # We want to avoid using the system Python because it requires root to use pip. - # python.org, MacPorts or HomeBrew Python installations should all be OK. - say "Installing python..." - $pkgcmd python - fi - - # Workaround for _dlopen not finding augeas on macOS - if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then - say "Applying augeas workaround" - mkdir -p /usr/local/lib/ - ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ - fi - - if ! hash pip 2>/dev/null; then - say "pip not installed" - say "Installing pip..." - curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python - fi - - if ! hash virtualenv 2>/dev/null; then - say "virtualenv not installed." - say "Installing with pip..." - pip install virtualenv - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh deleted file mode 100644 index dfa5b47f3..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh +++ /dev/null @@ -1,30 +0,0 @@ -# If new packages are installed by BootstrapMageiaCommon below, this version -# number must be increased. -BOOTSTRAP_MAGEIA_COMMON_VERSION=1 - -BootstrapMageiaCommon() { - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='--quiet' - fi - - if ! urpmi --force $QUIET_FLAG \ - python \ - libpython-devel \ - python-virtualenv - then - error "Could not install Python dependencies. Aborting bootstrap!" - exit 1 - fi - - if ! urpmi --force $QUIET_FLAG \ - git \ - gcc \ - python-augeas \ - libopenssl-devel \ - libffi-devel \ - rootcerts - then - error "Could not install additional dependencies. Aborting bootstrap!" - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh deleted file mode 100755 index 80d55a393..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ /dev/null @@ -1,45 +0,0 @@ -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 - - InitializeRPMCommonBase - - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $TOOL list python >/dev/null 2>&1; then - python_pkgs="$python - python-devel - python-virtualenv - python-tools - python-pip - " - # Fedora 26 starts to use the prefix python2 for python2 based packages. - # this elseif is theoretically for any Fedora over version 26: - elif $TOOL list python2 >/dev/null 2>&1; then - python_pkgs="$python2 - python2-libs - python2-setuptools - python2-devel - python2-virtualenv - python2-tools - python2-pip - " - # Some distros and older versions of current distros use a "python27" - # instead of the "python" or "python-" naming convention. - else - python_pkgs="$python27 - python27-devel - python27-virtualenv - python27-tools - python27-pip - " - fi - - BootstrapRpmCommonBase "$python_pkgs" -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh deleted file mode 100644 index 2b00b199b..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh +++ /dev/null @@ -1,60 +0,0 @@ -# If new packages are installed by BootstrapRpmCommonBase below, version -# numbers in rpm_common.sh and rpm_python3.sh must be increased. - -# Sets TOOL to the name of the package manager -# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. -# Note: this function is called both while selecting the bootstrap scripts and -# during the actual bootstrap. Some things like prompting to user can be done in the latter -# case, but not in the former one. -InitializeRPMCommonBase() { - if type dnf 2>/dev/null - then - TOOL=dnf - elif type yum 2>/dev/null - then - TOOL=yum - - else - error "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 - fi - - if [ "$ASSUME_YES" = 1 ]; then - YES_FLAG="-y" - fi - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='--quiet' - fi -} - -BootstrapRpmCommonBase() { - # Arguments: whitespace-delimited python packages to install - - InitializeRPMCommonBase # This call is superfluous in practice - - pkgs=" - gcc - augeas-libs - openssl - openssl-devel - libffi-devel - redhat-rpm-config - ca-certificates - " - - # Add the python packages - pkgs="$pkgs - $1 - " - - if $TOOL list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi - - if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh deleted file mode 100644 index ac0553db5..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh +++ /dev/null @@ -1,23 +0,0 @@ -# If new packages are installed by BootstrapRpmPython3 below, this version -# number must be increased. -BOOTSTRAP_RPM_PYTHON3_VERSION=1 - -BootstrapRpmPython3() { - # Tested with: - # - Fedora 29 - - InitializeRPMCommonBase - - # Fedora 29 must use python3-virtualenv - if $TOOL list python3-virtualenv >/dev/null 2>&1; then - python_pkgs="python3 - python3-virtualenv - python3-devel - " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 - fi - - BootstrapRpmCommonBase "$python_pkgs" -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh deleted file mode 100644 index febfc7a83..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh +++ /dev/null @@ -1,78 +0,0 @@ -# If new packages are installed by BootstrapRpmPython3 below, this version -# number must be increased. -BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION=1 - -# Checks if rh-python36 can be installed. -Python36SclIsAvailable() { - InitializeRPMCommonBase >/dev/null 2>&1; - - if "${TOOL}" list rh-python36 >/dev/null 2>&1; then - return 0 - fi - if "${TOOL}" list centos-release-scl >/dev/null 2>&1; then - return 0 - fi - return 1 -} - -# Try to enable rh-python36 from SCL if it is necessary and possible. -EnablePython36SCL() { - if "$EXISTS" python3.6 > /dev/null 2> /dev/null; then - return 0 - fi - if [ ! -f /opt/rh/rh-python36/enable ]; then - return 0 - fi - set +e - if ! . /opt/rh/rh-python36/enable; then - error 'Unable to enable rh-python36!' - exit 1 - fi - set -e -} - -# This bootstrap concerns old RedHat-based distributions that do not ship by default -# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing -# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6. -BootstrapRpmPython3Legacy() { - # Tested with: - # - CentOS 6 - - InitializeRPMCommonBase - - if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then - echo "To use Certbot on this operating system, packages from the SCL repository need to be installed." - if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then - error "Enable the SCL repository and try running Certbot again." - exit 1 - fi - if [ "${ASSUME_YES}" = 1 ]; then - /bin/echo -n "Enabling the SCL repository in 3 seconds... (Press Ctrl-C to cancel)" - sleep 1s - /bin/echo -ne "\e[0K\rEnabling the SCL repository in 2 seconds... (Press Ctrl-C to cancel)" - sleep 1s - /bin/echo -e "\e[0K\rEnabling the SCL repository in 1 second... (Press Ctrl-C to cancel)" - sleep 1s - fi - if ! "${TOOL}" install "${YES_FLAG}" "${QUIET_FLAG}" centos-release-scl; then - error "Could not enable SCL. Aborting bootstrap!" - exit 1 - fi - fi - - # CentOS 6 must use rh-python36 from SCL - if "${TOOL}" list rh-python36 >/dev/null 2>&1; then - python_pkgs="rh-python36-python - rh-python36-python-virtualenv - rh-python36-python-devel - " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 - fi - - BootstrapRpmCommonBase "${python_pkgs}" - - # Enable SCL rh-python36 after bootstrapping. - EnablePython36SCL -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh b/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh deleted file mode 100644 index ac7c0ed4a..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh +++ /dev/null @@ -1,8 +0,0 @@ -# If new packages are installed by BootstrapSmartOS below, this version number -# must be increased. -BOOTSTRAP_SMARTOS_VERSION=1 - -BootstrapSmartOS() { - pkgin update - pkgin -y install 'gcc49' 'py27-augeas' 'py27-virtualenv' -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh deleted file mode 100755 index 7fa28ce50..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh +++ /dev/null @@ -1,36 +0,0 @@ -# If new packages are installed by BootstrapSuseCommon below, this version -# number must be increased. -BOOTSTRAP_SUSE_COMMON_VERSION=1 - -BootstrapSuseCommon() { - # SLE12 don't have python-virtualenv - - if [ "$ASSUME_YES" = 1 ]; then - zypper_flags="-nq" - install_flags="-l" - fi - - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='-qq' - fi - - if zypper search -x python-virtualenv >/dev/null 2>&1; then - OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" - else - # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv - # is a source package, and python2-virtualenv must be used instead. - # Also currently python2-setuptools is not a dependency of python2-virtualenv, - # while it should be. Installing it explicitly until upstream fix. - OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" - fi - - zypper $QUIET_FLAG $zypper_flags in $install_flags \ - python \ - python-devel \ - $OPENSUSE_VIRTUALENV_PACKAGES \ - gcc \ - augeas-lenses \ - libopenssl-devel \ - libffi-devel \ - ca-certificates -} diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt deleted file mode 100644 index 4d4c91a5d..000000000 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b diff --git a/letsencrypt-auto-source/pieces/check_permissions.py b/letsencrypt-auto-source/pieces/check_permissions.py deleted file mode 100644 index ba55e6d97..000000000 --- a/letsencrypt-auto-source/pieces/check_permissions.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Verifies certbot-auto cannot be modified by unprivileged users. - -This script takes the path to certbot-auto as its only command line -argument. It then checks that the file can only be modified by uid/gid -< 1000 and if other users can modify the file, it prints a warning with -a suggestion on how to solve the problem. - -Permissions on symlinks in the absolute path of certbot-auto are ignored -and only the canonical path to certbot-auto is checked. There could be -permissions problems due to the symlinks that are unreported by this -script, however, issues like this were not caused by our documentation -and are ignored for the sake of simplicity. - -All warnings are printed to stdout rather than stderr so all stderr -output from this script can be suppressed to avoid printing messages if -this script fails for some reason. - -""" -from __future__ import print_function - -import os -import stat -import sys - - -FORUM_POST_URL = 'https://community.letsencrypt.org/t/certbot-auto-deployment-best-practices/91979/' - - -def has_safe_permissions(path): - """Returns True if the given path has secure permissions. - - The permissions are considered safe if the file is only writable by - uid/gid < 1000. - - The reason we allow more IDs than 0 is because on some systems such - as Debian, system users/groups other than uid/gid 0 are used for the - path we recommend in our instructions which is /usr/local/bin. 1000 - was chosen because on Debian 0-999 is reserved for system IDs[1] and - on RHEL either 0-499 or 0-999 is reserved depending on the - version[2][3]. Due to these differences across different OSes, this - detection isn't perfect so we only determine permissions are - insecure when we can be reasonably confident there is a problem - regardless of the underlying OS. - - [1] https://www.debian.org/doc/debian-policy/ch-opersys.html#uid-and-gid-classes - [2] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/ch-managing_users_and_groups - [3] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/system_administrators_guide/ch-managing_users_and_groups - - :param str path: filesystem path to check - :returns: True if the path has secure permissions, otherwise, False - :rtype: bool - - """ - # os.stat follows symlinks before obtaining information about a file. - stat_result = os.stat(path) - if stat_result.st_mode & stat.S_IWOTH: - return False - if stat_result.st_mode & stat.S_IWGRP and stat_result.st_gid >= 1000: - return False - if stat_result.st_mode & stat.S_IWUSR and stat_result.st_uid >= 1000: - return False - return True - - -def main(certbot_auto_path): - current_path = os.path.realpath(certbot_auto_path) - last_path = None - permissions_ok = True - # This loop makes use of the fact that os.path.dirname('/') == '/'. - while current_path != last_path and permissions_ok: - permissions_ok = has_safe_permissions(current_path) - last_path = current_path - current_path = os.path.dirname(current_path) - - if not permissions_ok: - print('{0} has insecure permissions!'.format(certbot_auto_path)) - print('To learn how to fix them, visit {0}'.format(FORUM_POST_URL)) - - -if __name__ == '__main__': - main(sys.argv[1]) diff --git a/letsencrypt-auto-source/pieces/create_venv.py b/letsencrypt-auto-source/pieces/create_venv.py deleted file mode 100755 index a618e228a..000000000 --- a/letsencrypt-auto-source/pieces/create_venv.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -import os -import shutil -import subprocess -import sys - - -def create_venv(venv_path, pyver, verbose): - if os.path.exists(venv_path): - shutil.rmtree(venv_path) - - stdout = sys.stdout if verbose == '1' else open(os.devnull, 'w') - - if int(pyver) <= 27: - # Use virtualenv binary - environ = os.environ.copy() - environ['VIRTUALENV_NO_DOWNLOAD'] = '1' - command = ['virtualenv', '--no-site-packages', '--python', sys.executable, venv_path] - subprocess.check_call(command, stdout=stdout, env=environ) - else: - # Use embedded venv module in Python 3 - command = [sys.executable, '-m', 'venv', venv_path] - subprocess.check_call(command, stdout=stdout) - - -if __name__ == '__main__': - create_venv(*sys.argv[1:]) diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt deleted file mode 100644 index f7a517e06..000000000 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ /dev/null @@ -1,264 +0,0 @@ -# This is the flattened list of packages certbot-auto installs. -# To generate this, do (with docker and package hashin installed): -# ``` -# letsencrypt-auto-source/rebuild_dependencies.py \ -# letsencrypt-auto-source/pieces/dependency-requirements.txt -# ``` -# If you want to update a single dependency, run commands similar to these: -# ``` -# pip install hashin -# hashin -r dependency-requirements.txt cryptography==1.5.2 -# ``` -ConfigArgParse==1.2.3 \ - --hash=sha256:edd17be986d5c1ba2e307150b8e5f5107aba125f3574dddd02c85d5cdcfd37dc -certifi==2020.4.5.1 \ - --hash=sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304 \ - --hash=sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519 -cffi==1.14.0 \ - --hash=sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff \ - --hash=sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b \ - --hash=sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac \ - --hash=sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0 \ - --hash=sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384 \ - --hash=sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26 \ - --hash=sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6 \ - --hash=sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b \ - --hash=sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e \ - --hash=sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd \ - --hash=sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2 \ - --hash=sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66 \ - --hash=sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc \ - --hash=sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8 \ - --hash=sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55 \ - --hash=sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4 \ - --hash=sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5 \ - --hash=sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d \ - --hash=sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78 \ - --hash=sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa \ - --hash=sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793 \ - --hash=sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f \ - --hash=sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a \ - --hash=sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f \ - --hash=sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30 \ - --hash=sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f \ - --hash=sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3 \ - --hash=sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c -chardet==3.0.4 \ - --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ - --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 -configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 -cryptography==2.8 \ - --hash=sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c \ - --hash=sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595 \ - --hash=sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad \ - --hash=sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651 \ - --hash=sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2 \ - --hash=sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff \ - --hash=sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d \ - --hash=sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42 \ - --hash=sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d \ - --hash=sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e \ - --hash=sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912 \ - --hash=sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793 \ - --hash=sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13 \ - --hash=sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7 \ - --hash=sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0 \ - --hash=sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879 \ - --hash=sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f \ - --hash=sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9 \ - --hash=sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2 \ - --hash=sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf \ - --hash=sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8 -distro==1.5.0 \ - --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ - --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 -enum34==1.1.10; python_version < '3.4' \ - --hash=sha256:a98a201d6de3f2ab3db284e70a33b0f896fbf35f8086594e8c9e74b909058d53 \ - --hash=sha256:c3858660960c984d6ab0ebad691265180da2b43f07e061c0f8dca9ef3cffd328 \ - --hash=sha256:cce6a7477ed816bd2542d03d53db9f0db935dd013b70f336a95c73979289f248 -funcsigs==1.0.2 \ - --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ - --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 -idna==2.9 \ - --hash=sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb \ - --hash=sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa -ipaddress==1.0.23 \ - --hash=sha256:6e0f4a39e66cb5bb9a137b00276a2eff74f93b71dcbdad6f10ff7df9d3557fcc \ - --hash=sha256:b7f8e0369580bb4a24d5ba1d7cc29660a4a6987763faf1d8a8046830e020e7e2 -josepy==1.3.0 \ - --hash=sha256:c341ffa403399b18e9eae9012f804843045764d1390f9cb4648980a7569b1619 \ - --hash=sha256:e54882c64be12a2a76533f73d33cba9e331950fda9e2731e843490b774e7a01c -mock==1.3.0 \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb -parsedatetime==2.5 \ - --hash=sha256:3b835fc54e472c17ef447be37458b400e3fefdf14bb1ffdedb5d2c853acf4ba1 \ - --hash=sha256:d2e9ddb1e463de871d32088a3f3cea3dc8282b1b2800e081bd0ef86900451667 -pbr==5.4.5 \ - --hash=sha256:07f558fece33b05caf857474a366dfcc00562bca13dd8b47b2b3e22d9f9bf55c \ - --hash=sha256:579170e23f8e0c2f24b0de612f71f648eccb79fb1322c814ae6b3c07b5ba23e8 -pyOpenSSL==19.1.0 \ - --hash=sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504 \ - --hash=sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507 -pyRFC3339==1.1 \ - --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ - --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a -pycparser==2.20 \ - --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ - --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 -pyparsing==2.4.7 \ - --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ - --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b -python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 -pytz==2020.1 \ - --hash=sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed \ - --hash=sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048 -requests==2.23.0 \ - --hash=sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee \ - --hash=sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6 -requests-toolbelt==0.9.1 \ - --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ - --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 -six==1.15.0 \ - --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ - --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced -urllib3==1.25.9 \ - --hash=sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527 \ - --hash=sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115 -zope.component==4.6.1 \ - --hash=sha256:bfbe55d4a93e70a78b10edc3aad4de31bb8860919b7cbd8d66f717f7d7b279ac \ - --hash=sha256:d9c7c27673d787faff8a83797ce34d6ebcae26a370e25bddb465ac2182766aca -zope.deferredimport==4.3.1 \ - --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \ - --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a -zope.deprecation==4.4.0 \ - --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ - --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 -zope.event==4.4 \ - --hash=sha256:69c27debad9bdacd9ce9b735dad382142281ac770c4a432b533d6d65c4614bcf \ - --hash=sha256:d8e97d165fd5a0997b45f5303ae11ea3338becfe68c401dd88ffd2113fe5cae7 -zope.hookable==5.0.1 \ - --hash=sha256:0194b9b9e7f614abba60c90b231908861036578297515d3d6508eb10190f266d \ - --hash=sha256:0c2977473918bdefc6fa8dfb311f154e7f13c6133957fe649704deca79b92093 \ - --hash=sha256:17b8bdb3b77e03a152ca0d5ca185a7ae0156f5e5a2dbddf538676633a1f7380f \ - --hash=sha256:29d07681a78042cdd15b268ae9decffed9ace68a53eebeb61d65ae931d158841 \ - --hash=sha256:36fb1b35d1150267cb0543a1ddd950c0bc2c75ed0e6e92e3aaa6ac2e29416cb7 \ - --hash=sha256:3aed60c2bb5e812bbf9295c70f25b17ac37c233f30447a96c67913ba5073642f \ - --hash=sha256:3cac1565cc768911e72ca9ec4ddf5c5109e1fef0104f19f06649cf1874943b60 \ - --hash=sha256:3d4bc0cc4a37c3cd3081063142eeb2125511db3c13f6dc932d899c512690378e \ - --hash=sha256:3f73096f27b8c28be53ffb6604f7b570fbbb82f273c6febe5f58119009b59898 \ - --hash=sha256:522d1153d93f2d48aa0bd9fb778d8d4500be2e4dcf86c3150768f0e3adbbc4ef \ - --hash=sha256:523d2928fb7377bbdbc9af9c0b14ad73e6eaf226349f105733bdae27efd15b5a \ - --hash=sha256:5848309d4fc5c02150a45e8f8d2227e5bfda386a508bbd3160fed7c633c5a2fa \ - --hash=sha256:6781f86e6d54a110980a76e761eb54590630fd2af2a17d7edf02a079d2646c1d \ - --hash=sha256:6fd27921ebf3aaa945fa25d790f1f2046204f24dba4946f82f5f0a442577c3e9 \ - --hash=sha256:70d581862863f6bf9e175e85c9d70c2d7155f53fb04dcdb2f73cf288ca559a53 \ - --hash=sha256:81867c23b0dc66c8366f351d00923f2bc5902820a24c2534dfd7bf01a5879963 \ - --hash=sha256:81db29edadcbb740cd2716c95a297893a546ed89db1bfe9110168732d7f0afdd \ - --hash=sha256:86bd12624068cea60860a0759af5e2c3adc89c12aef6f71cf12f577e28deefe3 \ - --hash=sha256:9c184d8f9f7a76e1ced99855ccf390ffdd0ec3765e5cbf7b9cada600accc0a1e \ - --hash=sha256:acc789e8c29c13555e43fe4bf9fcd15a65512c9645e97bbaa5602e3201252b02 \ - --hash=sha256:afaa740206b7660d4cc3b8f120426c85761f51379af7a5b05451f624ad12b0af \ - --hash=sha256:b5f5fa323f878bb16eae68ea1ba7f6c0419d4695d0248bed4b18f51d7ce5ab85 \ - --hash=sha256:bd89e0e2c67bf4ac3aca2a19702b1a37269fb1923827f68324ac2e7afd6e3406 \ - --hash=sha256:c212de743283ec0735db24ec6ad913758df3af1b7217550ff270038062afd6ae \ - --hash=sha256:ca553f524293a0bdea05e7f44c3e685e4b7b022cb37d87bc4a3efa0f86587a8d \ - --hash=sha256:cab67065a3db92f636128d3157cc5424a145f82d96fb47159c539132833a6d36 \ - --hash=sha256:d3b3b3eedfdbf6b02898216e85aa6baf50207f4378a2a6803d6d47650cd37031 \ - --hash=sha256:d9f4a5a72f40256b686d31c5c0b1fde503172307beb12c1568296e76118e402c \ - --hash=sha256:df5067d87aaa111ed5d050e1ee853ba284969497f91806efd42425f5348f1c06 \ - --hash=sha256:e2587644812c6138f05b8a41594a8337c6790e3baf9a01915e52438c13fc6bef \ - --hash=sha256:e27fd877662db94f897f3fd532ef211ca4901eb1a70ba456f15c0866a985464a \ - --hash=sha256:e427ebbdd223c72e06ba94c004bb04e996c84dec8a0fa84e837556ae145c439e \ - --hash=sha256:e583ad4309c203ef75a09d43434cf9c2b4fa247997ecb0dcad769982c39411c7 \ - --hash=sha256:e760b2bc8ece9200804f0c2b64d10147ecaf18455a2a90827fbec4c9d84f3ad5 \ - --hash=sha256:ea9a9cc8bcc70e18023f30fa2f53d11ae069572a162791224e60cd65df55fb69 \ - --hash=sha256:ecb3f17dce4803c1099bd21742cd126b59817a4e76a6544d31d2cca6e30dbffd \ - --hash=sha256:ed794e3b3de42486d30444fb60b5561e724ee8a2d1b17b0c2e0f81e3ddaf7a87 \ - --hash=sha256:ee885d347279e38226d0a437b6a932f207f691c502ee565aba27a7022f1285df \ - --hash=sha256:fd5e7bc5f24f7e3d490698f7b854659a9851da2187414617cd5ed360af7efd63 \ - --hash=sha256:fe45f6870f7588ac7b2763ff1ce98cce59369717afe70cc353ec5218bc854bcc -zope.interface==5.1.0 \ - --hash=sha256:0103cba5ed09f27d2e3de7e48bb320338592e2fabc5ce1432cf33808eb2dfd8b \ - --hash=sha256:14415d6979356629f1c386c8c4249b4d0082f2ea7f75871ebad2e29584bd16c5 \ - --hash=sha256:1ae4693ccee94c6e0c88a4568fb3b34af8871c60f5ba30cf9f94977ed0e53ddd \ - --hash=sha256:1b87ed2dc05cb835138f6a6e3595593fea3564d712cb2eb2de963a41fd35758c \ - --hash=sha256:269b27f60bcf45438e8683269f8ecd1235fa13e5411de93dae3b9ee4fe7f7bc7 \ - --hash=sha256:27d287e61639d692563d9dab76bafe071fbeb26818dd6a32a0022f3f7ca884b5 \ - --hash=sha256:39106649c3082972106f930766ae23d1464a73b7d30b3698c986f74bf1256a34 \ - --hash=sha256:40e4c42bd27ed3c11b2c983fecfb03356fae1209de10686d03c02c8696a1d90e \ - --hash=sha256:461d4339b3b8f3335d7e2c90ce335eb275488c587b61aca4b305196dde2ff086 \ - --hash=sha256:4f98f70328bc788c86a6a1a8a14b0ea979f81ae6015dd6c72978f1feff70ecda \ - --hash=sha256:558a20a0845d1a5dc6ff87cd0f63d7dac982d7c3be05d2ffb6322a87c17fa286 \ - --hash=sha256:562dccd37acec149458c1791da459f130c6cf8902c94c93b8d47c6337b9fb826 \ - --hash=sha256:5e86c66a6dea8ab6152e83b0facc856dc4d435fe0f872f01d66ce0a2131b7f1d \ - --hash=sha256:60a207efcd8c11d6bbeb7862e33418fba4e4ad79846d88d160d7231fcb42a5ee \ - --hash=sha256:645a7092b77fdbc3f68d3cc98f9d3e71510e419f54019d6e282328c0dd140dcd \ - --hash=sha256:6874367586c020705a44eecdad5d6b587c64b892e34305bb6ed87c9bbe22a5e9 \ - --hash=sha256:74bf0a4f9091131de09286f9a605db449840e313753949fe07c8d0fe7659ad1e \ - --hash=sha256:7b726194f938791a6691c7592c8b9e805fc6d1b9632a833b9c0640828cd49cbc \ - --hash=sha256:8149ded7f90154fdc1a40e0c8975df58041a6f693b8f7edcd9348484e9dc17fe \ - --hash=sha256:8cccf7057c7d19064a9e27660f5aec4e5c4001ffcf653a47531bde19b5aa2a8a \ - --hash=sha256:911714b08b63d155f9c948da2b5534b223a1a4fc50bb67139ab68b277c938578 \ - --hash=sha256:a5f8f85986197d1dd6444763c4a15c991bfed86d835a1f6f7d476f7198d5f56a \ - --hash=sha256:a744132d0abaa854d1aad50ba9bc64e79c6f835b3e92521db4235a1991176813 \ - --hash=sha256:af2c14efc0bb0e91af63d00080ccc067866fb8cbbaca2b0438ab4105f5e0f08d \ - --hash=sha256:b054eb0a8aa712c8e9030065a59b5e6a5cf0746ecdb5f087cca5ec7685690c19 \ - --hash=sha256:b0becb75418f8a130e9d465e718316cd17c7a8acce6fe8fe07adc72762bee425 \ - --hash=sha256:b1d2ed1cbda2ae107283befd9284e650d840f8f7568cb9060b5466d25dc48975 \ - --hash=sha256:ba4261c8ad00b49d48bbb3b5af388bb7576edfc0ca50a49c11dcb77caa1d897e \ - --hash=sha256:d1fe9d7d09bb07228650903d6a9dc48ea649e3b8c69b1d263419cc722b3938e8 \ - --hash=sha256:d7804f6a71fc2dda888ef2de266727ec2f3915373d5a785ed4ddc603bbc91e08 \ - --hash=sha256:da2844fba024dd58eaa712561da47dcd1e7ad544a257482392472eae1c86d5e5 \ - --hash=sha256:dcefc97d1daf8d55199420e9162ab584ed0893a109f45e438b9794ced44c9fd0 \ - --hash=sha256:dd98c436a1fc56f48c70882cc243df89ad036210d871c7427dc164b31500dc11 \ - --hash=sha256:e74671e43ed4569fbd7989e5eecc7d06dc134b571872ab1d5a88f4a123814e9f \ - --hash=sha256:eb9b92f456ff3ec746cd4935b73c1117538d6124b8617bc0fe6fda0b3816e345 \ - --hash=sha256:ebb4e637a1fb861c34e48a00d03cffa9234f42bef923aec44e5625ffb9a8e8f9 \ - --hash=sha256:ef739fe89e7f43fb6494a43b1878a36273e5924869ba1d866f752c5812ae8d58 \ - --hash=sha256:f40db0e02a8157d2b90857c24d89b6310f9b6c3642369852cdc3b5ac49b92afc \ - --hash=sha256:f68bf937f113b88c866d090fea0bc52a098695173fc613b055a17ff0cf9683b6 \ - --hash=sha256:fb55c182a3f7b84c1a2d6de5fa7b1a05d4660d866b91dbf8d74549c57a1499e8 -zope.proxy==4.3.5 \ - --hash=sha256:00573dfa755d0703ab84bb23cb6ecf97bb683c34b340d4df76651f97b0bab068 \ - --hash=sha256:092049280f2848d2ba1b57b71fe04881762a220a97b65288bcb0968bb199ec30 \ - --hash=sha256:0cbd27b4d3718b5ec74fc65ffa53c78d34c65c6fd9411b8352d2a4f855220cf1 \ - --hash=sha256:17fc7e16d0c81f833a138818a30f366696653d521febc8e892858041c4d88785 \ - --hash=sha256:19577dfeb70e8a67249ba92c8ad20589a1a2d86a8d693647fa8385408a4c17b0 \ - --hash=sha256:207aa914576b1181597a1516e1b90599dc690c095343ae281b0772e44945e6a4 \ - --hash=sha256:219a7db5ed53e523eb4a4769f13105118b6d5b04ed169a283c9775af221e231f \ - --hash=sha256:2b50ea79849e46b5f4f2b0247a3687505d32d161eeb16a75f6f7e6cd81936e43 \ - --hash=sha256:5903d38362b6c716e66bbe470f190579c530a5baf03dbc8500e5c2357aa569a5 \ - --hash=sha256:5c24903675e271bd688c6e9e7df5775ac6b168feb87dbe0e4bcc90805f21b28f \ - --hash=sha256:5ef6bc5ed98139e084f4e91100f2b098a0cd3493d4e76f9d6b3f7b95d7ad0f06 \ - --hash=sha256:61b55ae3c23a126a788b33ffb18f37d6668e79a05e756588d9e4d4be7246ab1c \ - --hash=sha256:63ddb992931a5e616c87d3d89f5a58db086e617548005c7f9059fac68c03a5cc \ - --hash=sha256:6943da9c09870490dcfd50c4909c0cc19f434fa6948f61282dc9cb07bcf08160 \ - --hash=sha256:6ad40f85c1207803d581d5d75e9ea25327cd524925699a83dfc03bf8e4ba72b7 \ - --hash=sha256:6b44433a79bdd7af0e3337bd7bbcf53dd1f9b0fa66bf21bcb756060ce32a96c1 \ - --hash=sha256:6bbaa245015d933a4172395baad7874373f162955d73612f0b66b6c2c33b6366 \ - --hash=sha256:7007227f4ea85b40a2f5e5a244479f6a6dfcf906db9b55e812a814a8f0e2c28d \ - --hash=sha256:74884a0aec1f1609190ec8b34b5d58fb3b5353cf22b96161e13e0e835f13518f \ - --hash=sha256:7d25fe5571ddb16369054f54cdd883f23de9941476d97f2b92eb6d7d83afe22d \ - --hash=sha256:7e162bdc5e3baad26b2262240be7d2bab36991d85a6a556e48b9dfb402370261 \ - --hash=sha256:814d62678dc3a30f4aa081982d830b7c342cf230ffc9d030b020cb154eeebf9e \ - --hash=sha256:8878a34c5313ee52e20aa50b03138af8d472bae465710fb954d133a9bfd3c38d \ - --hash=sha256:a66a0d94e5b081d5d695e66d6667e91e74d79e273eee95c1747717ba9cb70792 \ - --hash=sha256:a69f5cbf4addcfdf03dda564a671040127a6b7c34cf9fe4973582e68441b63fa \ - --hash=sha256:b00f9f0c334d07709d3f73a7cb8ae63c6ca1a90c790a63b5e7effa666ef96021 \ - --hash=sha256:b6ed71e4a7b4690447b626f499d978aa13197a0e592950e5d7020308f6054698 \ - --hash=sha256:bdf5041e5851526e885af579d2f455348dba68d74f14a32781933569a327fddf \ - --hash=sha256:be034360dd34e62608419f86e799c97d389c10a0e677a25f236a971b2f40dac9 \ - --hash=sha256:cc8f590a5eed30b314ae6b0232d925519ade433f663de79cc3783e4b10d662ba \ - --hash=sha256:cd7a318a15fe6cc4584bf3c4426f092ed08c0fd012cf2a9173114234fe193e11 \ - --hash=sha256:cf19b5f63a59c20306e034e691402b02055c8f4e38bf6792c23cad489162a642 \ - --hash=sha256:cfc781ce442ec407c841e9aa51d0e1024f72b6ec34caa8fdb6ef9576d549acf2 \ - --hash=sha256:dea9f6f8633571e18bc20cad83603072e697103a567f4b0738d52dd0211b4527 \ - --hash=sha256:e4a86a1d5eb2cce83c5972b3930c7c1eac81ab3508464345e2b8e54f119d5505 \ - --hash=sha256:e7106374d4a74ed9ff00c46cc00f0a9f06a0775f8868e423f85d4464d2333679 \ - --hash=sha256:e98a8a585b5668aa9e34d10f7785abf9545fe72663b4bfc16c99a115185ae6a5 \ - --hash=sha256:f64840e68483316eb58d82c376ad3585ca995e69e33b230436de0cdddf7363f9 \ - --hash=sha256:f8f4b0a9e6683e43889852130595c8854d8ae237f2324a053cdd884de936aa9b \ - --hash=sha256:fc45a53219ed30a7f670a6d8c98527af0020e6fd4ee4c0a8fb59f147f06d816c diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py deleted file mode 100644 index 1515fe353..000000000 --- a/letsencrypt-auto-source/pieces/fetch.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Do downloading and JSON parsing without additional dependencies. :: - - # Print latest released version of LE to stdout: - python fetch.py --latest-version - - # Download letsencrypt-auto script from git tag v1.2.3 into the folder I'm - # in, and make sure its signature verifies: - python fetch.py --le-auto-script v1.2.3 - -On failure, return non-zero. - -""" - -from __future__ import print_function, unicode_literals - -from distutils.version import LooseVersion -from json import loads -from os import devnull, environ -from os.path import dirname, join -import re -import ssl -from subprocess import check_call, CalledProcessError -from sys import argv, exit -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler - from urllib2 import HTTPError, URLError -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler - from urllib.error import HTTPError, URLError - -PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq -OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 -xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp -9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij -n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH -cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ -CQIDAQAB ------END PUBLIC KEY----- -""") - -class ExpectedError(Exception): - """A novice-readable exception that also carries the original exception for - debugging""" - - -class HttpsGetter(object): - def __init__(self): - """Build an HTTPS opener.""" - # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. - if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=cert_none_context())) - else: - self._opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in self._opener.handlers: - if isinstance(handler, HTTPHandler): - self._opener.handlers.remove(handler) - - def get(self, url): - """Return the document contents pointed to by an HTTPS URL. - - If something goes wrong (404, timeout, etc.), raise ExpectedError. - - """ - try: - # socket module docs say default timeout is None: that is, no - # timeout - return self._opener.open(url, timeout=30).read() - except (HTTPError, IOError) as exc: - raise ExpectedError("Couldn't download %s." % url, exc) - - -def write(contents, dir, filename): - """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'wb') as file: - file.write(contents) - - -def latest_stable_version(get): - """Return the latest stable release of letsencrypt.""" - metadata = loads(get( - environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) - # metadata['info']['version'] actually returns the latest of any kind of - # release release, contrary to https://wiki.python.org/moin/PyPIJSON. - # The regex is a sufficient regex for picking out prereleases for most - # packages, LE included. - return str(max(LooseVersion(r) for r - in metadata['releases'].keys() - if re.match('^[0-9.]+$', r))) - - -def verified_new_le_auto(get, tag, temp_dir): - """Return the path to a verified, up-to-date letsencrypt-auto script. - - If the download's signature does not verify or something else goes wrong - with the verification process, raise ExpectedError. - - """ - le_auto_dir = environ.get( - 'LE_AUTO_DIR_TEMPLATE', - 'https://raw.githubusercontent.com/certbot/certbot/%s/' - 'letsencrypt-auto-source/') % tag - write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') - write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') - try: - with open(devnull, 'w') as dev_null: - check_call(['openssl', 'dgst', '-sha256', '-verify', - join(temp_dir, 'public_key.pem'), - '-signature', - join(temp_dir, 'letsencrypt-auto.sig'), - join(temp_dir, 'letsencrypt-auto')], - stdout=dev_null, - stderr=dev_null) - except CalledProcessError as exc: - raise ExpectedError("Couldn't verify signature of downloaded " - "certbot-auto.", exc) - - -def cert_none_context(): - """Create a SSLContext object to not check hostname.""" - # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.verify_mode = ssl.CERT_NONE - return context - - -def main(): - get = HttpsGetter().get - flag = argv[1] - try: - if flag == '--latest-version': - print(latest_stable_version(get)) - elif flag == '--le-auto-script': - tag = argv[2] - verified_new_le_auto(get, tag, dirname(argv[0])) - except ExpectedError as exc: - print(exc.args[0], exc.args[1]) - return 1 - else: - return 0 - - -if __name__ == '__main__': - exit(main()) diff --git a/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt deleted file mode 100644 index 8e745c9cd..000000000 --- a/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Contains the requirements for the letsencrypt package. -# -# Since the letsencrypt package depends on certbot and using pip with hashes -# requires that all installed packages have hashes listed, this allows -# dependency-requirements.txt to be used without requiring a hash for a -# (potentially unreleased) Certbot package. - -letsencrypt==0.7.0 \ - --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ - --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py deleted file mode 100755 index 7610c2686..000000000 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python -"""A small script that can act as a trust root for installing pip >=8 -Embed this in your project, and your VCS checkout is all you have to trust. In -a post-peep era, this lets you claw your way to a hash-checking version of pip, -with which you can install the rest of your dependencies safely. All it assumes -is Python 2.6 or better and *some* version of pip already installed. If -anything goes wrong, it will exit with a non-zero status code. -""" -# This is here so embedded copies are MIT-compliant: -# Copyright (c) 2016 Erik Rose -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -from __future__ import print_function -from distutils.version import StrictVersion -from hashlib import sha256 -from os import environ -from os.path import join -from shutil import rmtree -try: - from subprocess import check_output -except ImportError: - from subprocess import CalledProcessError, PIPE, Popen - - def check_output(*popenargs, **kwargs): - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be ' - 'overridden.') - process = Popen(stdout=PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd) - return output -import sys -from tempfile import mkdtemp -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse # 3.4 - - -__version__ = 1, 5, 1 -PIP_VERSION = '9.0.1' -DEFAULT_INDEX_BASE = 'https://pypi.python.org' - - -# wheel has a conditional dependency on argparse: -maybe_argparse = ( - [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' - 'argparse-1.4.0.tar.gz', - '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if sys.version_info < (2, 7, 0) else []) - - -# Be careful when updating the pinned versions here, in particular for pip. -# Indeed starting from 10.0, pip will build dependencies in isolation if the -# related projects are compliant with PEP 517. This is not something we want -# as of now, so the isolation build will need to be disabled wherever -# pipstrap is used (see https://github.com/certbot/certbot/issues/8256). -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz'.format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), - # This version of setuptools has only optional dependencies: - ('37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/' - 'setuptools-40.6.3.zip', - '3b474dad69c49f0d2d86696b68105f3a6f195f7ab655af12ef9a9c326d2b08f8'), - ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' - 'wheel-0.29.0.tar.gz', - '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') -] - - -class HashError(Exception): - def __str__(self): - url, path, actual, expected = self.args - return ('{url} did not match the expected hash {expected}. Instead, ' - 'it was {actual}. The file (left at {path}) may have been ' - 'tampered with.'.format(**locals())) - - -def hashed_download(url, temp, digest): - """Download ``url`` to ``temp``, make sure it has the SHA-256 ``digest``, - and return its path.""" - # Based on pip 1.4.1's URLOpener but with cert verification removed. Python - # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert - # authenticity has only privacy (not arbitrary code execution) - # implications, since we're checking hashes. - def opener(using_https=True): - opener = build_opener(HTTPSHandler()) - if using_https: - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) - return opener - - def read_chunks(response, chunk_size): - while True: - chunk = response.read(chunk_size) - if not chunk: - break - yield chunk - - parsed_url = urlparse(url) - response = opener(using_https=parsed_url.scheme == 'https').open(url) - path = join(temp, parsed_url.path.split('/')[-1]) - actual_hash = sha256() - with open(path, 'wb') as file: - for chunk in read_chunks(response, 4096): - file.write(chunk) - actual_hash.update(chunk) - - actual_digest = actual_hash.hexdigest() - if actual_digest != digest: - raise HashError(url, path, actual_digest, digest) - return path - - -def get_index_base(): - """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the - end if it's there; that is likely to give us the right dir. - """ - env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') - if env_var: - SIMPLE = '/simple' - if env_var.endswith(SIMPLE): - return env_var[:-len(SIMPLE)] - else: - return env_var - else: - return DEFAULT_INDEX_BASE - - -def main(): - python = sys.executable or 'python' - pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) - .decode('utf-8').split()[1]) - has_pip_cache = pip_version >= StrictVersion('6.0') - index_base = get_index_base() - temp = mkdtemp(prefix='pipstrap-') - try: - downloads = [hashed_download(index_base + '/packages/' + path, - temp, - digest) - for path, digest in PACKAGES] - # Calling pip as a module is the preferred way to avoid problems about pip self-upgrade. - command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] - # Disable cache since it is not used and it otherwise sometimes throws permission warnings: - command.extend(['--no-cache-dir'] if has_pip_cache else []) - command.extend(downloads) - check_output(command) - except HashError as exc: - print(exc) - except Exception: - rmtree(temp) - raise - else: - rmtree(temp) - return 0 - return 1 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/letsencrypt-auto-source/rebuild_dependencies.py b/letsencrypt-auto-source/rebuild_dependencies.py deleted file mode 100755 index 864394661..000000000 --- a/letsencrypt-auto-source/rebuild_dependencies.py +++ /dev/null @@ -1,290 +0,0 @@ -#!/usr/bin/env python -""" -Gather and consolidate the up-to-date dependencies available and required to install certbot -on various Linux distributions. It generates a requirements file contained the pinned and hashed -versions, ready to be used by pip to install the certbot dependencies. - -This script is typically used to update the certbot-requirements.txt file of certbot-auto. - -To achieve its purpose, this script will start a certbot installation with unpinned dependencies, -then gather them, on various distributions started as Docker containers. - -Usage: letsencrypt-auto-source/rebuild_dependencies new_requirements.txt - -NB1: Docker must be installed on the machine running this script. -NB2: Python library 'hashin' must be installed on the machine running this script. -""" -from __future__ import print_function -import re -import shutil -import subprocess -import tempfile -import os -from os.path import dirname, abspath, join -import sys -import argparse - -# The list of docker distributions to test dependencies against with. -DISTRIBUTION_LIST = [ - 'ubuntu:18.04', 'ubuntu:16.04', - 'debian:stretch', - 'centos:7', 'centos:6', - 'opensuse/leap:15', - 'fedora:29', -] - -# These constraints will be added while gathering dependencies on each distribution. -# It can be used because a particular version for a package is required for any reason, -# or to solve a version conflict between two distributions requirements. -AUTHORITATIVE_CONSTRAINTS = { - # Using an older version of mock here prevents regressions of #5276. - 'mock': '1.3.0', - # Too touchy to move to a new version. And will be removed soon - # in favor of pure python parser for Apache. - 'python-augeas': '0.5.0', - # Package enum34 needs to be explicitly limited to Python2.x, in order to avoid - # certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456. - 'enum34': '1.1.10; python_version < \'3.4\'', - # Cryptography 2.9+ drops support for OpenSSL 1.0.1, but we still want to support it - # for officially supported non-x86_64 ancient distributions like RHEL 6. - 'cryptography': '2.8', - # Parsedatetime 2.6 is broken on Python 2.7, see https://github.com/bear/parsedatetime/issues/246 - 'parsedatetime': '2.5', -} - -# ./certbot/letsencrypt-auto-source/rebuild_dependencies.py (2 levels from certbot root path) -CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__))) - -# The script will be used to gather dependencies for a given distribution. -# - certbot-auto is used to install relevant OS packages, and set up an initial venv -# - then this venv is used to consistently construct an empty new venv -# - once pipstraped, this new venv pip-installs certbot runtime (including apache/nginx), -# without pinned dependencies, and respecting input authoritative requirements -# - `certbot plugins` is called to check we have a healthy environment -# - finally current set of dependencies is extracted out of the docker using pip freeze -SCRIPT = r"""#!/bin/sh -set -e - -cd /tmp/certbot -letsencrypt-auto-source/letsencrypt-auto --install-only -n -PYVER=`/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - -/opt/eff.org/certbot/venv/bin/python letsencrypt-auto-source/pieces/create_venv.py /tmp/venv "$PYVER" 1 - -/tmp/venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py -/tmp/venv/bin/pip install -e acme -e certbot -e certbot-apache -e certbot-nginx -c /tmp/constraints.txt -/tmp/venv/bin/certbot plugins -/tmp/venv/bin/pip freeze >> /tmp/workspace/requirements.txt -""" - - -def _read_from(file): - """Read all content of the file, and return it as a string.""" - with open(file, 'r') as file_h: - return file_h.read() - - -def _write_to(file, content): - """Write given string content to the file, overwriting its initial content.""" - with open(file, 'w') as file_h: - file_h.write(content) - - -def _requirements_from_one_distribution(distribution, verbose): - """ - Calculate the Certbot dependencies expressed for the given distribution, using the official - Docker for this distribution, and return the lines of the generated requirements file. - """ - print('===> Gathering dependencies for {0}.'.format(distribution)) - workspace = tempfile.mkdtemp() - script = join(workspace, 'script.sh') - authoritative_constraints = join(workspace, 'constraints.txt') - cid_file = join(workspace, 'cid') - - try: - _write_to(script, SCRIPT) - os.chmod(script, 0o755) - - _write_to(authoritative_constraints, '\n'.join( - '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) - - command = ['docker', 'run', '--rm', '--cidfile', cid_file, - '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), - '-v', '{0}:/tmp/workspace'.format(workspace), - '-v', '{0}:/tmp/constraints.txt'.format(authoritative_constraints), - distribution, '/tmp/workspace/script.sh'] - sub_stdout = sys.stdout if verbose else subprocess.PIPE - sub_stderr = sys.stderr if verbose else subprocess.STDOUT - process = subprocess.Popen(command, stdout=sub_stdout, stderr=sub_stderr, universal_newlines=True) - stdoutdata, _ = process.communicate() - - if process.returncode: - if stdoutdata: - sys.stderr.write('Output was:\n{0}'.format(stdoutdata)) - raise RuntimeError('Error while gathering dependencies for {0}.'.format(distribution)) - - with open(join(workspace, 'requirements.txt'), 'r') as file_h: - return file_h.readlines() - finally: - if os.path.isfile(cid_file): - cid = _read_from(cid_file) - try: - subprocess.check_output(['docker', 'kill', cid], stderr=subprocess.PIPE) - except subprocess.CalledProcessError: - pass - shutil.rmtree(workspace) - - -def _parse_and_merge_requirements(dependencies_map, requirements_file_lines, distribution): - """ - Extract every requirement from the given requirements file, and merge it in the dependency map. - Merging here means that the map contain every encountered dependency, and the version used in - each distribution. - - Example: - # dependencies_map = { - # } - _parse_and_merge_requirements(['cryptography=='1.2','requests=='2.1.0'], dependencies_map, 'debian:stretch') - # dependencies_map = { - # 'cryptography': [('1.2', 'debian:stretch)], - # 'requests': [('2.1.0', 'debian:stretch')] - # } - _parse_and_merge_requirements(['requests=='2.4.0', 'mock==1.3'], dependencies_map, 'centos:7') - # dependencies_map = { - # 'cryptography': [('1.2', 'debian:stretch)], - # 'requests': [('2.1.0', 'debian:stretch'), ('2.4.0', 'centos:7')], - # 'mock': [('2.4.0', 'centos:7')] - # } - """ - for line in requirements_file_lines: - match = re.match(r'([^=]+)==([^=]+)', line.strip()) - if not line.startswith('-e') and match: - package, version = match.groups() - if package not in ['acme', 'certbot', 'certbot-apache', 'certbot-nginx', 'pkg-resources']: - dependencies_map.setdefault(package, []).append((version, distribution)) - - -def _consolidate_and_validate_dependencies(dependency_map): - """ - Given the dependency map of all requirements found in all distributions for Certbot, - construct an array containing the unit requirements for Certbot to be used by pip, - and the version conflicts, if any, between several distributions for a package. - Return requirements and conflicts as a tuple. - """ - print('===> Consolidate and validate the dependency map.') - requirements = [] - conflicts = [] - for package, versions in dependency_map.items(): - reduced_versions = _reduce_versions(versions) - - if len(reduced_versions) > 1: - version_list = ['{0} ({1})'.format(version, ','.join(distributions)) - for version, distributions in reduced_versions.items()] - conflict = ('package {0} is declared with several versions: {1}' - .format(package, ', '.join(version_list))) - conflicts.append(conflict) - sys.stderr.write('ERROR: {0}\n'.format(conflict)) - else: - requirements.append((package, list(reduced_versions)[0])) - - requirements.sort(key=lambda x: x[0]) - return requirements, conflicts - - -def _reduce_versions(version_dist_tuples): - """ - Get an array of version/distribution tuples, - and reduce it to a map based on the version values. - - Example: [('1.2.0', 'debian:stretch'), ('1.4.0', 'ubuntu:18.04'), ('1.2.0', 'centos:6')] - => {'1.2.0': ['debiqn:stretch', 'centos:6'], '1.4.0': ['ubuntu:18.04']} - """ - version_dist_map = {} - for version, distribution in version_dist_tuples: - version_dist_map.setdefault(version, []).append(distribution) - - return version_dist_map - - -def _write_requirements(dest_file, requirements, conflicts): - """ - Given the list of requirements and conflicts, write a well-formatted requirements file, - whose requirements are hashed signed using hashin library. Conflicts are written at the end - of the generated file. - """ - print('===> Calculating hashes for the requirement file.') - - _write_to(dest_file, '''\ -# This is the flattened list of packages certbot-auto installs. -# To generate this, do (with docker and package hashin installed): -# ``` -# letsencrypt-auto-source/rebuild_dependencies.py \\ -# letsencrypt-auto-source/pieces/dependency-requirements.txt -# ``` -# If you want to update a single dependency, run commands similar to these: -# ``` -# pip install hashin -# hashin -r dependency-requirements.txt cryptography==1.5.2 -# ``` -''') - - for req in requirements: - if req[0] in AUTHORITATIVE_CONSTRAINTS: - # If requirement is in AUTHORITATIVE_CONSTRAINTS, take its value instead of the - # computed one to get any environment descriptor that would have been added. - req = (req[0], AUTHORITATIVE_CONSTRAINTS[req[0]]) - subprocess.check_call(['hashin', '{0}=={1}'.format(req[0], req[1]), - '--requirements-file', dest_file]) - - if conflicts: - with open(dest_file, 'a') as file_h: - file_h.write('\n## ! SOME ERRORS OCCURRED ! ##\n') - file_h.write('\n'.join('# {0}'.format(conflict) for conflict in conflicts)) - file_h.write('\n') - - return _read_from(dest_file) - - -def _gather_dependencies(dest_file, verbose): - """ - Main method of this script. Given a destination file path, will write the file - containing the consolidated and hashed requirements for Certbot, validated - against several Linux distributions. - """ - dependencies_map = {} - - for distribution in DISTRIBUTION_LIST: - requirements_file_lines = _requirements_from_one_distribution(distribution, verbose) - _parse_and_merge_requirements(dependencies_map, requirements_file_lines, distribution) - - requirements, conflicts = _consolidate_and_validate_dependencies(dependencies_map) - - return _write_requirements(dest_file, requirements, conflicts) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=('Build a sanitized, pinned and hashed requirements file for certbot-auto, ' - 'validated against several OS distributions using Docker.')) - parser.add_argument('requirements_path', - help='path for the generated requirements file') - parser.add_argument('--verbose', '-v', action='store_true', - help='verbose will display all output during docker execution') - - namespace = parser.parse_args() - - try: - subprocess.check_output(['hashin', '--version']) - except subprocess.CalledProcessError: - raise RuntimeError('Python library hashin is not installed in the current environment.') - - try: - subprocess.check_output(['docker', '--version'], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - raise RuntimeError('Docker is not installed or accessible to current user.') - - file_content = _gather_dependencies(namespace.requirements_path, namespace.verbose) - - print(file_content) - print('===> Rebuilt requirement file is available on path {0}' - .format(abspath(namespace.requirements_path))) diff --git a/letsencrypt-auto-source/tests/signing.key b/letsencrypt-auto-source/tests/signing.key deleted file mode 100644 index b9964d00c..000000000 --- a/letsencrypt-auto-source/tests/signing.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEAsMoSzLYQ7E1sdSOkwelgtzKIh2qi3bpXuYtcfFC0XrvWig07 -1NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7GhFW0VdbxL6JdGzS2ShNWkX9hE9z+ -j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTTuUtJmmGcuk3a9Aq/sCT6DdfmTSdP -5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVglLsIVPBuy9IcgHidUQ96hJnoPsDCW -sHwX62495QKEarauyKQrJzFes0EY95orDM47Z5o/NDiQB11m91yNB0MmPYY9QSbn -OA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68iQIDAQABAoIBAQCJE3W2Mqk2f+XL -geKa1BjAkzcXQJCduYGRhUQlw/HGzoBPtGki56Tf53MeHTAkIGfIq3CAr1zRhiNv -8SQzvrLQIx/buvhxhcQJdzqsfwgNcqXT3/OliF34P3LMx8GUfPy/6xq2Qdv4fvwA -nLJH8wyDTKP6RxtdvUY7GSZ+Ln2QQv/3Nco7tax4GHNGom8iSgeH/YKTDnvitdqh -a0fr930QzU39TfOftLmasdmKUOIg8G2wr4Sy6Kn060+OUoQr1fZF5mnLvvQeILCK -uav91JkIeMLggzk+t88IJUFWdOoxv5hWTnNzHyt+/GYfovyRz2fKQMwzdh1F8iM5 -+867rEb9AoGBANn1ncemJBedDshStdCBUH0+2ExPrawveaXOZKnx8/VGFXNi0hAf -KzkntMWd5g5kB077FtKO9CYTBvK4pZBWIFLcJEqAz88JeXME6dfUbRucDr72ko+l -rcLHXj7F0IDVzj/9CphMGAhC9J/4YW9SPcSbMw6dQ6xOk73f1Vowve0DAoGBAM+k -/F+hVqCS3f22Bg9KuDtx+zCydaZxC842DgIkV1SO2iFhNHjnpQ5EIR0WrSYeV2n+ -rD7kVs5OH1HvnGScHaQKtAVqZClSwF14jzE+Aj8XDwxiHLSOhJgKlzfVX7h1ymMh -7fsslDl6xNGQ+40gubhkCLT5qABFKy1mrZ8b+3yDAoGAGLGUI6d2FVrM7vM3+Bx+ -gwIYvWSVl5l1XcypaPupmRNMoNsEU6FEY2BVQcJm6yB4F4GpD0f0709ejSdQUq7/ -UIPydKJtaNZ49QgMelBt4B/pJ8eFyVKLAjNWQSRmQAJ5MJS5m5Gbc2wqjOk2GMen -idvPiAtXPHFWmb9/S42UJwMCgYEAjymAe2qgcGtyNNfIC8kHhqzKdEPGi/ALJKzu -MZnewEURrcv4QpfrnA9rCUQ2Mz7eJA1bsqz6EJmaTIK4wEFGynA6uDUnQ7pzOL7D -cz7+i4MZc/89LVvJnY5Hvk4WBfboiDq/etq8g3jatGaSmTYD9la6DhTHORB3eYD+ -meHQHYMCgYEA18y9hnx2k4vNeBei4YXF4pAvKdwKLQD+CcP9ljb3VT+kXktjRA1C -aWj3HhMwvcxtttfkQzEnwwGRAkTEtNewJ8KFxhmc9nYElZTNZ+SuHD5Dkv8xqoj8 -NvG8rU1eiEyPwE2wQxpM5JLqbo7IWtR0dmptjKoF1gRxn6Wh4TwEiHA= ------END RSA PRIVATE KEY----- diff --git a/tests/letstest/README.md b/letstest/README.md similarity index 69% rename from tests/letstest/README.md rename to letstest/README.md index 76db57153..c569d1e8f 100644 --- a/tests/letstest/README.md +++ b/letstest/README.md @@ -14,17 +14,13 @@ Simple AWS testfarm scripts for certbot client testing are needed, they need to be requested via online webform. ## Installation and configuration -These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you're -on a Debian based system, make sure you also have the python3-venv package -installed. If you have Python 3 installed, you can use requirements.txt to -create a virtual environment with a known set of dependencies by running: -``` -python3 -m venv venv3 -. ./venv3/bin/activate -pip install --requirement requirements.txt -``` -You can then configure AWS credentials and create a key by running: +This package is installed in the Certbot development environment that is +created by following the instructions at +https://certbot.eff.org/docs/contributing.html#running-a-local-copy-of-the-client. + +After activating that virtual environment, you can then configure AWS +credentials and create a key by running: ``` >aws configure --profile [interactive: enter secrets for IAM role] @@ -35,9 +31,9 @@ Note: whatever you pick for `` will be shown to other users with AWS a When prompted for a default region name, enter: `us-east-1`. ## Usage -To run tests, activate the virtual environment you created above and run: +To run tests, activate the virtual environment you created above and from this directory run: ``` ->python multitester.py targets.yaml /path/to/your/key.pem scripts/ +>letstest targets/targets.yaml /path/to/your/key.pem scripts/ ``` You can only run up to two tests at once. The following error is often indicative of there being too many AWS instances running on our account: @@ -52,15 +48,14 @@ aws ec2 terminate-instances --profile --instance-ids $(aws ec2 de It will take a minute for these instances to shut down and become available again. Running this will invalidate any in progress tests. -A folder named `letest-` is also created with a log file from each instance of the test and a file named "results" containing the output above. +A temporary directory whose name is output by the tests is also created with a log file from each instance of the test and a file named "results" containing the output above. The tests take quite a while to run. ## Scripts Example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed to them at runtime via environment variables. test_apache2.sh is a useful reference. -Note that the 
test_letsencrypt_auto_*
scripts pull code from PyPI using the letsencrypt-auto script, -__not__ the local python code. test_apache2 runs the dev venv and does local tests. +test_apache2 runs the dev venv and does local tests. See: - https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html diff --git a/letstest/letstest/__init__.py b/letstest/letstest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/letstest/multitester.py b/letstest/letstest/multitester.py similarity index 94% rename from tests/letstest/multitester.py rename to letstest/letstest/multitester.py index 83a92e6dc..a56bf0f37 100644 --- a/tests/letstest/multitester.py +++ b/letstest/letstest/multitester.py @@ -22,21 +22,18 @@ Usage: >aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \ --query 'KeyMaterial' --output text > MyKeyPair.pem then: ->python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_leauto_upgrades.sh +>letstest targets/targets.yaml MyKeyPair.pem HappyHacker scripts/test_sdists.sh see: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html """ - -from __future__ import print_function -from __future__ import with_statement - import argparse import multiprocessing as mp from multiprocessing import Manager import os import socket import sys +import tempfile import time import traceback import urllib.error as urllib_error @@ -49,7 +46,6 @@ import yaml from fabric import Config from fabric import Connection - # Command line parser #------------------------------------------------------------------------------- parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.') @@ -60,7 +56,7 @@ parser.add_argument('key_file', parser.add_argument('aws_profile', help='profile for AWS (i.e. as in ~/.aws/certificates)') parser.add_argument('test_script', - default='test_letsencrypt_auto_certonly_standalone.sh', + default='test_sdists.sh', help='path of bash script in to deploy and run') parser.add_argument('--repo', default='https://github.com/letsencrypt/letsencrypt.git', @@ -95,7 +91,7 @@ SECURITY_GROUP_NAME = 'certbot-security-group' SENTINEL = None #queue kill signal SUBNET_NAME = 'certbot-subnet' -class Status(object): +class Status: """Possible statuses of client tests.""" PASS = 'pass' FAIL = 'fail' @@ -195,18 +191,16 @@ def block_until_ssh_open(ipstring, wait_time=10, timeout=120): t_elapsed += wait_time sock.close() -def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20): +def block_until_instance_ready(booting_instance, extra_wait_time=20): "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections" - state = booting_instance.state['Name'] - ip = booting_instance.public_ip_address - while state != 'running' or ip is None: - time.sleep(wait_time) - # The instance needs to be reloaded to update its local attributes. See - # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Instance.reload. - booting_instance.reload() - state = booting_instance.state['Name'] - ip = booting_instance.public_ip_address - block_until_ssh_open(ip) + booting_instance.wait_until_running() + # The instance needs to be reloaded to update its local attributes. See + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Instance.reload. + booting_instance.reload() + # After waiting for the instance to be running and reloading the instance + # state, we should have an IP address. + assert booting_instance.public_ip_address is not None + block_until_ssh_open(booting_instance.public_ip_address) time.sleep(extra_wait_time) return booting_instance @@ -381,9 +375,8 @@ def main(): # Set up local copy of git repo #------------------------------------------------------------------------------- - log_dir = "letest-%d"%int(time.time()) #points to logging / working directory - print("Making local dir for test repo and logs: %s"%log_dir) - local_cxn.local('mkdir %s'%log_dir) + log_dir = tempfile.mkdtemp() # points to logging / working directory + print("Local dir for test repo and logs: %s"%log_dir) try: # figure out what git object to test and locally create it in log_dir @@ -503,12 +496,17 @@ def main(): outputs = [outq for outq in iter(outqueue.get, SENTINEL)] outputs.sort(key=lambda x: x[0]) failed = False + results_msg = "" for outq in outputs: ii, target, status = outq if status == Status.FAIL: failed = True - print('%d %s %s'%(ii, target['name'], status)) + with open(log_dir+'/'+'%d_%s.log'%(ii,target['name']), 'r') as f: + print(target['name'] + " test failed. Test log:") + print(f.read()) + results_msg = results_msg + '%d %s %s\n'%(ii, target['name'], status) results_file.write('%d %s %s\n'%(ii, target['name'], status)) + print(results_msg) if len(outputs) != num_processes: failed = True failure_message = 'FAILURE: Some target machines failed to run and were not tested. ' +\ diff --git a/tests/letstest/scripts/bootstrap_os_packages.sh b/letstest/scripts/bootstrap_os_packages.sh similarity index 76% rename from tests/letstest/scripts/bootstrap_os_packages.sh rename to letstest/scripts/bootstrap_os_packages.sh index 7ad93f63e..3f4c6e30e 100755 --- a/tests/letstest/scripts/bootstrap_os_packages.sh +++ b/letstest/scripts/bootstrap_os_packages.sh @@ -34,9 +34,9 @@ DeterminePythonVersion() { } BootstrapDebCommon() { - apt-get update || error apt-get update hit problems but continuing anyway... + sudo apt-get update || error apt-get update hit problems but continuing anyway... - apt-get install -y --no-install-recommends \ + sudo apt-get install -y --no-install-recommends \ python3 \ python3-dev \ python3-venv \ @@ -46,8 +46,19 @@ BootstrapDebCommon() { openssl \ libffi-dev \ ca-certificates \ + build-essential \ + curl \ make # needed on debian 9 arm64 which doesn't have a python3 pynacl wheel + # make sure rust isn't installed by the package manager + if ! sudo apt-get remove -y rustc; then + error "Could not remove existing rust. Aborting bootstrap!" + exit 1 + fi + + # Install rust for cryptography (needed on Debian) + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + . $HOME/.cargo/env } # Sets TOOL to the name of the package manager @@ -79,6 +90,7 @@ BootstrapRpmCommonBase() { libffi-devel redhat-rpm-config ca-certificates + cargo " # Add the python packages @@ -92,7 +104,7 @@ BootstrapRpmCommonBase() { " fi - if ! $TOOL install -y $pkgs; then + if ! sudo $TOOL install -y $pkgs; then error "Could not install OS dependencies. Aborting bootstrap!" exit 1 fi @@ -105,8 +117,8 @@ BootstrapRpmPython3() { python3-devel " - if ! $TOOL list 'python3*-devel' >/dev/null 2>&1; then - yum-config-manager --enable rhui-REGION-rhel-server-extras rhui-REGION-rhel-server-optional + if ! sudo $TOOL list 'python3*-devel' >/dev/null 2>&1; then + sudo yum-config-manager --enable rhui-REGION-rhel-server-extras rhui-REGION-rhel-server-optional fi BootstrapRpmCommonBase "$python_pkgs" diff --git a/tests/letstest/scripts/test_apache2.sh b/letstest/scripts/test_apache2.sh similarity index 96% rename from tests/letstest/scripts/test_apache2.sh rename to letstest/scripts/test_apache2.sh index 77dc35f1e..9d9ca6c12 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/letstest/scripts/test_apache2.sh @@ -59,7 +59,7 @@ fi cd letsencrypt echo "Bootstrapping dependencies..." -sudo tests/letstest/scripts/bootstrap_os_packages.sh +sudo letstest/scripts/bootstrap_os_packages.sh if [ $? -ne 0 ] ; then exit 1 fi @@ -113,7 +113,7 @@ elif [ "$OS_TYPE" = "centos" ]; then fi OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9) APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8) -"venv/bin/python" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" +"venv/bin/python" letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" if [ $? -ne 0 ] ; then FAIL=1 fi diff --git a/tests/letstest/scripts/test_openssl_version.py b/letstest/scripts/test_openssl_version.py similarity index 100% rename from tests/letstest/scripts/test_openssl_version.py rename to letstest/scripts/test_openssl_version.py diff --git a/letstest/scripts/test_sdists.sh b/letstest/scripts/test_sdists.sh new file mode 100755 index 000000000..562169524 --- /dev/null +++ b/letstest/scripts/test_sdists.sh @@ -0,0 +1,53 @@ +#!/bin/sh -xe + +cd letsencrypt + +BOOTSTRAP_SCRIPT="letstest/scripts/bootstrap_os_packages.sh" +VENV_PATH=venv + +# install OS packages +. $BOOTSTRAP_SCRIPT + +# setup venv +python3 -m venv $VENV_PATH +$VENV_PATH/bin/python3 tools/pipstrap.py +. "$VENV_PATH/bin/activate" +# pytest is needed to run tests on our packages so we install a pinned version here. +tools/pip_install.py pytest + +# setup constraints +TEMP_DIR=$(mktemp -d) +CONSTRAINTS="$TEMP_DIR/constraints.txt" +cp tools/requirements.txt "$CONSTRAINTS" + +# We pin cryptography to 3.1.1 and pyopenssl to 19.1.0 specifically for CentOS 7 / RHEL 7 +# because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been +# dropped on cryptography>=3.2 and pyopenssl>=20.0.0. +# Using this old version of OpenSSL would break the cryptography and pyopenssl wheels builds. +if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then + sed -i 's|cryptography==.*|cryptography==3.1.1|g' "$CONSTRAINTS" + sed -i 's|pyopenssl==.*|pyopenssl==19.1.0|g' "$CONSTRAINTS" +fi + + +PLUGINS="certbot-apache certbot-nginx" +# build sdists +for pkg_dir in acme certbot $PLUGINS; do + cd $pkg_dir + python setup.py clean + rm -rf build dist + python setup.py sdist + mv dist/* $TEMP_DIR + cd - +done + +VERSION=$(python letstest/scripts/version.py) +# test sdists +cd $TEMP_DIR +for pkg in acme certbot $PLUGINS; do + tar -xvf "$pkg-$VERSION.tar.gz" + cd "$pkg-$VERSION" + PIP_CONSTRAINT=../constraints.txt PIP_NO_BINARY=:all: pip install . + python -m pytest + cd - +done diff --git a/letsencrypt-auto-source/version.py b/letstest/scripts/version.py similarity index 75% rename from letsencrypt-auto-source/version.py rename to letstest/scripts/version.py index d70ffefac..6e538b032 100755 --- a/letsencrypt-auto-source/version.py +++ b/letstest/scripts/version.py @@ -1,8 +1,7 @@ #!/usr/bin/env python """Get the current Certbot version number. -Provides simple utilities for determining the Certbot version number and -building letsencrypt-auto. +Provides a simple utility for determining the Certbot version number """ from __future__ import print_function @@ -10,10 +9,10 @@ from os.path import abspath, dirname, join import re -def certbot_version(build_script_dir): +def certbot_version(letstest_scripts_dir): """Return the version number stamped in certbot/__init__.py.""" return re.search('''^__version__ = ['"](.+)['"].*''', - file_contents(join(dirname(build_script_dir), + file_contents(join(dirname(dirname(letstest_scripts_dir)), 'certbot', 'certbot', '__init__.py')), diff --git a/letstest/setup.py b/letstest/setup.py new file mode 100644 index 000000000..a552cf920 --- /dev/null +++ b/letstest/setup.py @@ -0,0 +1,45 @@ +from setuptools import find_packages +from setuptools import setup + +setup( + name='letstest', + version='1.0', + description='Test Certbot on different AWS images', + url='https://github.com/certbot/certbot', + author='Certbot Project', + author_email='certbot-dev@eff.org', + license='Apache License 2.0', + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=[ + # awscli isn't required by the tests themselves, but it is a useful + # tool to have when using these tests to generate keys and control + # running instances so the dependency is declared here for convenience. + 'awscli', + 'boto3', + 'botocore', + # The API from Fabric 2.0+ is used instead of the 1.0 API. + 'fabric>=2', + 'pyyaml', + ], + entry_points={ + 'console_scripts': [ + 'letstest=letstest.multitester:main', + ], + } +) diff --git a/tests/letstest/apache2_targets.yaml b/letstest/targets/apache2_targets.yaml similarity index 100% rename from tests/letstest/apache2_targets.yaml rename to letstest/targets/apache2_targets.yaml diff --git a/tests/letstest/targets.yaml b/letstest/targets/targets.yaml similarity index 100% rename from tests/letstest/targets.yaml rename to letstest/targets/targets.yaml diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index c9061ecb3..b3f1349e0 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -70,13 +70,21 @@ parts: - python3-pkg-resources - python3.8-minimal # To build cryptography and cffi if needed - build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev] + build-packages: + - gcc + - git + - libaugeas-dev + - build-essential + - libssl-dev + - libffi-dev + - python3-dev + - cargo build-environment: - SNAPCRAFT_PYTHON_VENV_ARGS: --upgrade - # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the + # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the # parts.[part_name].constraints option available in snapcraft.yaml when the Python plugin is # used. This is done to let these constraints be applied not only on the certbot package - # build, but also on any isolated build that pip could trigger when building wheels for + # build, but also on any isolated build that pip could trigger when building wheels for # dependencies. See https://github.com/certbot/certbot/pull/8443 for more info. - PIP_CONSTRAINT: $SNAPCRAFT_PART_SRC/snap-constraints.txt override-build: | @@ -85,9 +93,7 @@ parts: snapcraftctl build override-pull: | snapcraftctl pull - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/letsencrypt-auto-source/pieces/dependency-requirements.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" - echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + grep -v python-augeas "${SNAPCRAFT_PART_SRC}/tools/requirements.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` shared-metadata: plugin: dump diff --git a/tests/letstest/auto_targets.yaml b/tests/letstest/auto_targets.yaml deleted file mode 100644 index 01d410227..000000000 --- a/tests/letstest/auto_targets.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# These images are located in us-east-1. - -targets: - #----------------------------------------------------------------------------- - #Ubuntu - - ami: ami-095192256fe1477ad - name: ubuntu18.04LTS - type: ubuntu - virt: hvm - user: ubuntu - - ami: ami-09677e0a6b14905b0 - name: ubuntu16.04LTS - type: ubuntu - virt: hvm - user: ubuntu - #----------------------------------------------------------------------------- - # Debian - - ami: ami-01db78123b2b99496 - name: debian10 - type: ubuntu - virt: hvm - user: admin - - ami: ami-003f19e0e687de1cd - name: debian9 - type: ubuntu - virt: hvm - user: admin - - ami: ami-0ed54dd1b25657636 - name: debian9_arm64 - type: ubuntu - virt: hvm - user: admin - machine_type: a1.medium - #----------------------------------------------------------------------------- - # Other Redhat Distros - - ami: ami-0916c408cb02e310b - name: RHEL7 - type: centos - virt: hvm - user: ec2-user - - ami: ami-0c322300a1dd5dc79 - name: RHEL8 - type: centos - virt: hvm - user: ec2-user - #----------------------------------------------------------------------------- - # CentOS - # These Marketplace AMIs must, irritatingly, have their terms manually - # agreed to on the AWS marketplace site for any new AWS account using them... - - ami: ami-9887c6e7 - name: centos7 - type: centos - virt: hvm - user: centos - - ami: ami-01ca03df4a6012157 - name: centos8 - type: centos - virt: hvm - user: centos diff --git a/tests/letstest/requirements.txt b/tests/letstest/requirements.txt deleted file mode 100644 index d30c507cf..000000000 --- a/tests/letstest/requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -awscli==1.18.88 -bcrypt==3.1.7 -boto3==1.14.11 -botocore==1.17.11 -cffi==1.14.0 -colorama==0.4.3 -cryptography==2.8 -docutils==0.15.2 -enum34==1.1.9 -fabric==2.5.0 -invoke==1.4.1 -ipaddress==1.0.23 -jmespath==0.9.5 -paramiko==2.7.1 -pyasn1==0.4.8 -pycparser==2.19 -PyNaCl==1.3.0 -python-dateutil==2.8.1 -PyYAML==5.3 -rsa==3.4.2 -s3transfer==0.3.3 -six==1.14.0 -urllib3==1.25.8 diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh deleted file mode 100755 index 407a865f2..000000000 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash -xe -set -o pipefail - -# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME -# are dynamically set at execution - -cd letsencrypt - -if ! command -v git ; then - if [ "$OS_TYPE" = "ubuntu" ] ; then - sudo apt-get update - fi - if ! ( sudo apt-get install -y git || sudo yum install -y git-all || sudo yum install -y git || sudo dnf install -y git ) ; then - echo git installation failed! - exit 1 - fi -fi -# If we're on a RHEL 6 based system, we can be confident Python is already -# installed because the package manager is written in Python. -if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then - # 0.20.0 is the latest version of letsencrypt-auto that doesn't install - # Python 3 on RHEL 6. - INITIAL_VERSION="0.20.0" - RUN_RHEL6_TESTS=1 -else - # 0.39.0 is the oldest version of letsencrypt-auto that works on CentOS 8. - INITIAL_VERSION="0.39.0" -fi - -git checkout -f "v$INITIAL_VERSION" letsencrypt-auto -if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 | grep "^certbot $INITIAL_VERSION$" ; then - echo initial installation appeared to fail - exit 1 -fi - -if command -v python; then - PYTHON_NAME="python" -else - PYTHON_NAME="python3" -fi - -# Now that python and openssl have been installed, we can set up a fake server -# to provide a new version of letsencrypt-auto. First, we start the server and -# directory to be served. -MY_TEMP_DIR=$(mktemp -d) -PORT_FILE="$MY_TEMP_DIR/port" -LOG_FILE="$MY_TEMP_DIR/log" -SERVER_PATH=$("$PYTHON_NAME" tools/readlink.py tools/simple_http_server.py) -cd "$MY_TEMP_DIR" -# We set PYTHONUNBUFFERED to disable buffering of output to LOG_FILE -PYTHONUNBUFFERED=1 "$PYTHON_NAME" "$SERVER_PATH" 0 > $PORT_FILE 2> "$LOG_FILE" & -SERVER_PID=$! -trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT -cd ~- - -# Then, we set up the files to be served. -FAKE_VERSION_NUM="99.99.99" -echo "{\"releases\": {\"$FAKE_VERSION_NUM\": null}}" > "$MY_TEMP_DIR/json" -LE_AUTO_SOURCE_DIR="$MY_TEMP_DIR/v$FAKE_VERSION_NUM" -NEW_LE_AUTO_PATH="$LE_AUTO_SOURCE_DIR/letsencrypt-auto" -mkdir "$LE_AUTO_SOURCE_DIR" -cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_SOURCE_DIR/letsencrypt-auto" -SIGNING_KEY="letsencrypt-auto-source/tests/signing.key" -openssl dgst -sha256 -sign "$SIGNING_KEY" -out "$NEW_LE_AUTO_PATH.sig" "$NEW_LE_AUTO_PATH" - -# Next, we wait for the server to start and get the port number. -sleep 5s -SERVER_PORT=$(sed -n 's/.*port \([0-9]\+\).*/\1/p' "$PORT_FILE") - -# Finally, we set the necessary certbot-auto environment variables. -export LE_AUTO_DIR_TEMPLATE="http://localhost:$SERVER_PORT/%s/" -export LE_AUTO_JSON_URL="http://localhost:$SERVER_PORT/json" -export LE_AUTO_PUBLIC_KEY="-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg -tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G -hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT -uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl -LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 -Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 -iQIDAQAB ------END PUBLIC KEY----- -" - -if [ "$RUN_RHEL6_TESTS" = 1 ]; then - if command -v python3; then - echo "Didn't expect Python 3 to be installed!" - exit 1 - fi - cp letsencrypt-auto cb-auto - if ! ./cb-auto -v --debug --version 2>&1 | grep "$INITIAL_VERSION" ; then - echo "Certbot shouldn't have updated to a new version!" - exit 1 - fi - # Create a 2nd venv at the old path to ensure we properly handle the (unlikely) case of two separate virtual environments below. - HOME=${HOME:-~root} - XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} - OLD_VENV_PATH="$XDG_DATA_HOME/letsencrypt" - export VENV_PATH="$OLD_VENV_PATH" - if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 | grep "^certbot $INITIAL_VERSION$" ; then - echo second installation appeared to fail - exit 1 - fi - unset VENV_PATH -fi - -if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then - echo "Had problems checking for updates!" - exit 1 -fi - -# Since certbot-auto is deprecated, we expect it to leave existing Certbot -# installations unmodified so we check for the same version that was initially -# installed below. -EXPECTED_VERSION="$INITIAL_VERSION" - -if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then - echo unexpected certbot version found - exit 1 -fi - -if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then - echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ - exit 1 -fi - -# Now let's test if letsencrypt-auto still tries to upgrade to a new version. -# Regardless of the OS, versions of the script with development version numbers -# ending in .dev0 will not upgrade. See -# https://github.com/certbot/certbot/blob/bdfb9f19c4086a60ef010d2431768850c26d838a/certbot-auto#L1947-L1948. -# In order to test the process of different OSes setting NO_SELF_UPGRADE as -# part of the script's deprecation, we make use of the fact that -# letsencrypt-auto should still attempt to fetch the version number from PyPI -# even if it has a development version number unless NO_SELF_UPGRADE is set in -# which case all of that logic should be skipped. -# -# First we make a copy of the current server logs. -PREVIOUS_LOG_FILE="$MY_TEMP_DIR/previous-log" -cp "$LOG_FILE" "$PREVIOUS_LOG_FILE" - -# Next we run letsencrypt-auto and make sure there were no problems checking -# for updates, the Certbot install still works, the version number is what -# we expect, and it prints a message about not receiving updates. -if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then - echo "Had problems checking for updates!" - exit 1 -fi -if ! ./letsencrypt-auto -v --debug --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then - echo unexpected certbot version found - exit 1 -fi -if ! ./letsencrypt-auto -v --debug --version 2>&1 | grep "will no longer receive updates" ; then - echo script did not print warning about not receiving updates! - exit 1 -fi - -# Finally, we check if our local server received more requests. Over time, -# we'll move more and more OSes into this case until it this is the expected -# behavior on all systems. -if [ -f /etc/redhat-release ]; then - if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then - echo our local server received unexpected requests - exit 1 - fi -else - if diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then - echo our local server did not receive the requests we expected - exit 1 - fi -fi diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh deleted file mode 100755 index 9573ab690..000000000 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -x -set -eo pipefail - -# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME are dynamically set at execution - -# with curl, instance metadata available from EC2 metadata service: -#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) -#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) -#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) - -cd letsencrypt -LE_AUTO_DIR="/usr/local/bin" -LE_AUTO_PATH="$LE_AUTO_DIR/letsencrypt-auto" -sudo cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_PATH" -sudo chown root "$LE_AUTO_PATH" -sudo chmod 0755 "$LE_AUTO_PATH" -export PATH="$LE_AUTO_DIR:$PATH" - -# Since certbot-auto is deprecated, we expect certbot-auto to error and -# refuse to install Certbot. -set +o pipefail -if ! letsencrypt-auto --debug --version | grep "Certbot cannot be installed."; then - echo "letsencrypt-auto didn't report being uninstallable." - exit 1 -fi -if [ ${PIPESTATUS[0]} != 1 ]; then - echo "letsencrypt-auto didn't exit with status 1 as expected" - exit 1 -fi diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh deleted file mode 100755 index aa12d5610..000000000 --- a/tests/letstest/scripts/test_sdists.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -xe - -cd letsencrypt - -BOOTSTRAP_SCRIPT="tests/letstest/scripts/bootstrap_os_packages.sh" -VENV_PATH=venv - -# install OS packages -sudo $BOOTSTRAP_SCRIPT - -# setup venv -# We strip the hashes because the venv creation script includes unhashed -# constraints in the commands given to pip and the mix of hashed and unhashed -# packages makes pip error out. -python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > requirements.txt -# We also strip out the requirement for enum34 because it cannot be installed -# in newer versions of Python 3, tools/strip_hashes.py removes the environment -# marker that'd normally prevent it from being installed, and this package is -# not needed for any OS tested here. -sed -i '/enum34/d' requirements.txt -CERTBOT_PIP_NO_BINARY=:all: tools/venv.py --requirement requirements.txt -. "$VENV_PATH/bin/activate" -# pytest is needed to run tests on some of our packages so we install a pinned version here. -tools/pip_install.py pytest - -PLUGINS="certbot-apache certbot-nginx" -TEMP_DIR=$(mktemp -d) - -# build sdists -for pkg_dir in acme certbot $PLUGINS; do - cd $pkg_dir - python setup.py clean - rm -rf build dist - python setup.py sdist - mv dist/* $TEMP_DIR - cd - -done - -VERSION=$(python letsencrypt-auto-source/version.py) -# test sdists -cd $TEMP_DIR -for pkg in acme certbot $PLUGINS; do - tar -xvf "$pkg-$VERSION.tar.gz" - cd "$pkg-$VERSION" - python setup.py build - python -m pytest - python setup.py install - cd - -done diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh deleted file mode 100755 index 858fc1f18..000000000 --- a/tests/letstest/scripts/test_tests.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -xe -# -# This script is useful for testing that the packages we've built for a release -# work on a variety of systems. For an example of the kinds of problems that -# can occur, see https://github.com/certbot/certbot/issues/3455. - -REPO_ROOT="letsencrypt" -LE_AUTO="$REPO_ROOT/letsencrypt-auto-source/letsencrypt-auto" -LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" -MODULES="acme certbot certbot-apache certbot-nginx" -PIP_INSTALL="tools/pip_install.py" -VENV_NAME=venv -BOOTSTRAP_SCRIPT="$REPO_ROOT/tests/letstest/scripts/bootstrap_os_packages.sh" -VENV_SCRIPT="tools/venv.py" - -sudo $BOOTSTRAP_SCRIPT - -cd $REPO_ROOT -$VENV_SCRIPT -. $VENV_NAME/bin/activate -"$PIP_INSTALL" pytest - -# To run tests that aren't packaged in modules, run pytest -# from the repo root. The directory structure should still -# cause the installed packages to be tested while using -# the tests available in the subdirectories. - -for module in $MODULES ; do - echo testing $module - pytest -v $module -done diff --git a/tests/lock_test.py b/tests/lock_test.py index dda318131..5035fcb37 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -1,6 +1,4 @@ """Tests to ensure the lock order is preserved.""" -from __future__ import print_function - import atexit import datetime import functools diff --git a/tests/modification-check.py b/tests/modification-check.py index 7a69fb1db..8f3ae1264 100755 --- a/tests/modification-check.py +++ b/tests/modification-check.py @@ -1,125 +1,58 @@ #!/usr/bin/env python +"""Ensures there have been no changes to important certbot-auto files.""" -from __future__ import print_function - +import hashlib import os -import shutil -import subprocess -import sys -import tempfile -try: - from urllib.request import urlretrieve -except ImportError: - from urllib import urlretrieve -def find_repo_path(): +# Relative to the root of the Certbot repo, these files are expected to exist +# and have the SHA-256 hashes contained in this dictionary. These hashes were +# taken from our v1.14.0 tag which was the last release we intended to make +# changes to certbot-auto. +# +# certbot-auto, letsencrypt-auto, and letsencrypt-auto-source/certbot-auto.asc +# can be removed from this dict after coordinating with tech ops to ensure we +# get the behavior we want from https://dl.eff.org. See +# https://github.com/certbot/certbot/issues/8742 for more info. +# +# Deleting letsencrypt-auto-source/letsencrypt-auto and +# letsencrypt-auto-source/letsencrypt-auto.sig can be done once we're +# comfortable breaking any certbot-auto scripts that haven't already updated to +# the last version. See +# https://opensource.eff.org/eff-open-source/pl/65geri7c4tr6iqunc1rpb3mpna for +# more info. +EXPECTED_FILES = { + 'certbot-auto': + 'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2', + 'letsencrypt-auto': + 'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2', + os.path.join('letsencrypt-auto-source', 'letsencrypt-auto'): + 'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2', + os.path.join('letsencrypt-auto-source', 'certbot-auto.asc'): + '0558ba7bd816732b38c092e8fedb6033dad01f263e290ec6b946263aaf6625a8', + os.path.join('letsencrypt-auto-source', 'letsencrypt-auto.sig'): + '61c036aabf75da350b0633da1b2bef0260303921ecda993455ea5e6d3af3b2fe', +} + + +def find_repo_root(): return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -# We do not use filecmp.cmp to take advantage of universal newlines -# handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. -# As a consequence, this function will not work correctly if executed by Python 2.x on Windows. -# But it will work correctly on Linux for any version, because every file tested will be LF. -def compare_files(path_1, path_2): - l1 = l2 = True - with open(path_1, 'r') as f1, open(path_2, 'r') as f2: - line = 1 - while l1 and l2: - line += 1 - l1 = f1.readline() - l2 = f2.readline() - if l1 != l2: - print('---') - print(( - 'While comparing {0} (1) and {1} (2), a difference was found at line {2}:' - .format(os.path.basename(path_1), os.path.basename(path_2), line))) - print('(1): {0}'.format(repr(l1))) - print('(2): {0}'.format(repr(l2))) - print('---') - return False - return True +def sha256_hash(filename): + hash_object = hashlib.sha256() + with open(filename, 'rb') as f: + hash_object.update(f.read()) + return hash_object.hexdigest() -def validate_scripts_content(repo_path, temp_cwd): - errors = False - - if not compare_files( - os.path.join(repo_path, 'certbot-auto'), - os.path.join(repo_path, 'letsencrypt-auto')): - print('Root certbot-auto and letsencrypt-auto differ.') - errors = True - else: - shutil.copyfile( - os.path.join(repo_path, 'certbot-auto'), - os.path.join(temp_cwd, 'local-auto')) - shutil.copy(os.path.normpath(os.path.join( - repo_path, - 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) - - # Compare file against current version in the target branch - branch = os.environ.get('TARGET_BRANCH', 'master') - url = ( - 'https://raw.githubusercontent.com/certbot/certbot/{0}/certbot-auto' - .format(branch)) - urlretrieve(url, os.path.join(temp_cwd, 'certbot-auto')) - - if compare_files( - os.path.join(temp_cwd, 'certbot-auto'), - os.path.join(temp_cwd, 'local-auto')): - print('Root *-auto were unchanged') - else: - # Compare file against the latest released version - latest_version = subprocess.check_output( - [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) - subprocess.check_call( - [sys.executable, 'fetch.py', '--le-auto-script', - 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) - if compare_files( - os.path.join(temp_cwd, 'letsencrypt-auto'), - os.path.join(temp_cwd, 'local-auto')): - print('Root *-auto were updated to the latest version.') - else: - print('Root *-auto have unexpected changes.') - errors = True - - return errors def main(): - repo_path = find_repo_path() - temp_cwd = tempfile.mkdtemp() - errors = False + repo_root = find_repo_root() + for filename, expected_hash in EXPECTED_FILES.items(): + filepath = os.path.join(repo_root, filename) + assert sha256_hash(filepath) == expected_hash, f'unexpected changes to {filepath}' + print('All certbot-auto files have correct hashes.') - try: - errors = validate_scripts_content(repo_path, temp_cwd) - - shutil.copyfile( - os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), - os.path.join(temp_cwd, 'original-lea') - ) - subprocess.check_call([sys.executable, os.path.normpath(os.path.join( - repo_path, 'letsencrypt-auto-source/build.py'))]) - shutil.copyfile( - os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), - os.path.join(temp_cwd, 'build-lea') - ) - shutil.copyfile( - os.path.join(temp_cwd, 'original-lea'), - os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')) - ) - - if not compare_files( - os.path.join(temp_cwd, 'original-lea'), - os.path.join(temp_cwd, 'build-lea')): - print('Script letsencrypt-auto-source/letsencrypt-auto ' - 'doesn\'t match output of build.py.') - errors = True - else: - print('Script letsencrypt-auto-source/letsencrypt-auto matches output of build.py.') - finally: - shutil.rmtree(temp_cwd) - - return errors if __name__ == '__main__': - if main(): - sys.exit(1) + main() diff --git a/tools/_release.sh b/tools/_release.sh index e9e0e49f0..4e118c2d7 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -30,9 +30,6 @@ echo Releasing production version "$version"... nextversion="$2" RELEASE_BRANCH="candidate-$version" -if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then - RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" -fi RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} # Needed to fix problems with git signatures and pinentry export GPG_TTY=$(tty) @@ -40,14 +37,13 @@ export GPG_TTY=$(tty) # port for a local Python Package Index (used in testing) PORT=${PORT:-1234} -# subpackages to be released (the way developers think about them) -SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" - # subpackages to be released (the way the script thinks about them) -SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" -SUBPKGS_NO_CERTBOT="$SUBPKGS_IN_AUTO_NO_CERTBOT $SUBPKGS_NOT_IN_AUTO" -SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" +SUBPKGS_NO_CERTBOT="acme certbot-apache certbot-nginx certbot-dns-cloudflare certbot-dns-cloudxns \ + certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy \ + certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns \ + certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 \ + certbot-dns-sakuracloud" +SUBPKGS="certbot $SUBPKGS_NO_CERTBOT" # certbot_compatibility_test is not packaged because: # - it is not meant to be used by anyone else than Certbot devs # - it causes problems when running pytest - the latter tries to @@ -181,77 +177,10 @@ cd ~- CERTBOT_DOCS=1 certbot --help all > certbot/docs/cli-help.txt jws --help > acme/docs/jws-help.txt -cd .. -# freeze before installing anything else, so that we know end-user KGS -# make sure "twine upload" doesn't catch "kgs" -if [ -d kgs ] ; then - echo Deleting old kgs... - rm -rf kgs -fi -mkdir kgs -kgs="kgs/$version" -pip freeze | tee $kgs -python ../tools/pip_install.py pytest -cd ~- -for module in $SUBPKGS ; do - echo testing $module - # use an empty configuration file rather than the one in the repo root - pytest -c <(echo '') $module -done - -# pin pip hashes of the things we just built -for pkg in $SUBPKGS_IN_AUTO ; do - echo $pkg==$version \\ - pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python -c 'from sys import stdin; input = stdin.read(); print(" ", input.replace("\n--hash", " \\\n --hash"), end="")' -done > letsencrypt-auto-source/pieces/certbot-requirements.txt deactivate -# there should be one requirement specifier and two hashes for each subpackage -expected_count=$(expr $(echo $SUBPKGS_IN_AUTO | wc -w) \* 3) -if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*$expected_count " ; then - echo Unexpected pip hash output - exit 1 -fi -# ensure we have the latest built version of leauto -letsencrypt-auto-source/build.py - -# Now we have to sign the built version of leauto. -SignLEAuto() { - yubico-piv-tool -a verify-pin --sign -s 9c -i letsencrypt-auto-source/letsencrypt-auto -o letsencrypt-auto-source/letsencrypt-auto.sig -} - -# Loop until letsencrypt-auto is signed correctly. -SignLEAuto || true -while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ - letsencrypt-auto-source/letsencrypt-auto.sig \ - letsencrypt-auto-source/letsencrypt-auto ; do - echo "The signature on letsencrypt-auto is not correct." - read -p "Would you like this script to try and sign it again [Y/n]?" response - case $response in - [yY][eE][sS]|[yY]|"") - SignLEAuto || true;; - *) - ;; - esac -done - -# This signature is not quite as strong, but easier for people to verify out of band -while ! gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto; do - echo "Unable to sign letsencrypt-auto using $RELEASE_KEY." - echo "Make sure your OpenPGP card is in your computer if you are using one." - echo "You may need to take the card out and put it back in again." - read -p "Press enter to try signing again." -done -# We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, -# but we can use the right name for certbot-auto.asc from day one -mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc - -# copy leauto to the root, overwriting the previous release version -cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto -cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto - -git add certbot-auto letsencrypt-auto letsencrypt-auto-source certbot/docs/cli-help.txt +git add certbot/docs/cli-help.txt while ! git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"; do echo "Unable to sign the release commit using git." echo "You may have to configure git to use gpg2 by running:" @@ -282,16 +211,12 @@ git add certbot/CHANGELOG.md git commit -m "Add contents to certbot/CHANGELOG.md for next version" echo "New root: $root" -echo "Test commands (in the letstest repo):" -echo 'python multitester.py auto_targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' -echo 'python multitester.py auto_targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' -echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' +echo "Test commands (in the letstest directory):" +echo 'letstest --saveinstances targets/targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' echo "In order to upload packages run the following command:" echo twine upload "$root/dist.$version/*/*" if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then SetVersion "$nextversion".dev0 - letsencrypt-auto-source/build.py - git add letsencrypt-auto-source/letsencrypt-auto git commit -m "Bump version to $nextversion" fi diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index f5140f9c7..f6bdea58c 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,7 +1,7 @@ # Specifies Python package versions for development and building Docker images. -# It includes in particular packages not specified in letsencrypt-auto's requirements file. +# It includes in particular packages not specified in tools/certbot_constraints.txt. # Some dev package versions specified here may be overridden by higher level constraints -# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). +# files during tests (eg. tools/oldest_constraints.txt). alabaster==0.7.10 apacheconfig==0.3.2 apipkg==1.4 @@ -16,8 +16,8 @@ backports.functools-lru-cache==1.5 backports.shutil-get-terminal-size==1.0.0 backports.ssl-match-hostname==3.7.0.1 bcrypt==3.1.6 -boto3==1.11.7 -botocore==1.14.7 +boto3==1.17.4 +botocore==1.20.4 cached-property==1.5.1 cloudflare==2.3.1 configparser==3.7.4 @@ -51,15 +51,16 @@ jedi==0.17.1 Jinja2==2.9.6 jmespath==0.9.4 josepy==1.1.0 +jsonpickle==2.0.0 jsonschema==2.6.0 lazy-object-proxy==1.4.3 logger==1.4 logilab-common==1.4.1 -MarkupSafe==1.0 +MarkupSafe==1.1.1 mccabe==0.6.1 more-itertools==5.0.0 msrest==0.6.18 -mypy==0.710 +mypy==0.812 mypy-extensions==0.4.3 ndg-httpsclient==0.3.2 oauth2client==4.0.0 @@ -82,10 +83,10 @@ PyGithub==1.52 Pygments==2.2.0 pyjwt==1.7.1 pylint==2.4.3 +pynacl==1.3.0 # If pynsist version is upgraded, our NSIS template windows-installer/template.nsi # must be upgraded if necessary using the new built-in one from pynsist. -pynacl==1.3.0 -pynsist==2.6 +pynsist==2.7 pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 @@ -93,7 +94,7 @@ pytest-xdist==1.22.5 pytest-sugar==0.9.2 pytest-rerunfailures==4.2 python-dateutil==2.8.1 -python-digitalocean==1.11 +python-digitalocean==1.15.0 python-dotenv==0.14.0 pywin32==300 PyYAML==5.3.1 @@ -119,6 +120,7 @@ traitlets==4.3.3 twine==1.11.0 typed-ast==1.4.1 typing==3.6.4 +typing-extensions==3.7.4.3 uritemplate==3.0.0 virtualenv==16.6.2 wcwidth==0.1.8 diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index 0d3626853..14bd3323a 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -14,10 +14,6 @@ WORKDIR /opt/certbot # Copy certbot code COPY CHANGELOG.md README.rst src/ -# We keep the relative path to the requirements file the same because, as of -# writing this, tools/pip_install.py is used in the Dockerfile for Certbot -# plugins and this script expects to find the requirements file there. -COPY letsencrypt-auto-source/pieces/dependency-requirements.txt letsencrypt-auto-source/pieces/ COPY tools tools COPY acme src/acme COPY certbot src/certbot @@ -31,18 +27,17 @@ RUN apk add --no-cache --virtual .certbot-deps \ binutils # Install certbot from sources -# -# We don't use tools/pip_install.py below so the hashes in -# dependency-requirements.txt can be used when installing packages for extra -# security. RUN apk add --no-cache --virtual .build-deps \ gcc \ linux-headers \ openssl-dev \ musl-dev \ libffi-dev \ + python3-dev \ + cargo \ && python tools/pipstrap.py \ && python tools/pip_install.py --no-cache-dir \ --editable src/acme \ --editable src/certbot \ - && apk del .build-deps + && apk del .build-deps \ + && rm -rf ${HOME}/.cargo diff --git a/tools/eff-pubkey.pem b/tools/eff-pubkey.pem deleted file mode 100644 index fe6c2f5bb..000000000 --- a/tools/eff-pubkey.pem +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq -OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 -xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp -9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij -n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH -cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ -CQIDAQAB ------END PUBLIC KEY----- diff --git a/tools/extract_changelog.py b/tools/extract_changelog.py index fb0b849aa..bd718191a 100755 --- a/tools/extract_changelog.py +++ b/tools/extract_changelog.py @@ -24,7 +24,7 @@ def main(): i = 0 while i < len(lines): if section_pattern.match(lines[i]): - i = i + 1 + i = i + 2 while i < len(lines): if NEW_SECTION_PATTERN.match(lines[i]): break @@ -32,8 +32,6 @@ def main(): i = i + 1 i = i + 1 - changelog = [entry for entry in changelog if entry] - print('\n'.join(changelog)) diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 0b47fa5f8..e7a34286c 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -5,8 +5,6 @@ # set to 1, packages are installed using pinned versions of all of our # dependencies. See pip_install.py for more information on the versions pinned # to. -from __future__ import print_function - import os import re import subprocess diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index bbcb38051..44a61249b 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -6,8 +6,6 @@ Only the simple formats SomeProject==1.2.3 or SomeProject<=1.2.3 are currently supported. """ -from __future__ import print_function - import sys diff --git a/tools/pinning/pin.sh b/tools/pinning/pin.sh new file mode 100755 index 000000000..ce47f02a0 --- /dev/null +++ b/tools/pinning/pin.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# This script accepts no arguments and automates the process of updating +# Certbot's dependencies. Dependencies can be pinned to older versions by +# modifying pyproject.toml in the same directory as this file. +set -euo pipefail + +WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +REPO_ROOT="$(dirname "$(dirname "${WORK_DIR}")")" +PIPSTRAP_CONSTRAINTS="${REPO_ROOT}/tools/pipstrap_constraints.txt" +RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")" +REQUIREMENTS_FILE="$REPO_ROOT/tools/requirements.txt" +STRIP_HASHES="${REPO_ROOT}/tools/strip_hashes.py" + +if ! command -v poetry >/dev/null; then + echo "Please install poetry." + echo "You may need to recreate Certbot's virtual environment and activate it." + exit 1 +fi + +cd "${WORK_DIR}" + +if [ -f poetry.lock ]; then + rm poetry.lock +fi + +poetry lock + +TEMP_REQUIREMENTS=$(mktemp) +trap 'rm poetry.lock; rm $TEMP_REQUIREMENTS' EXIT + +poetry export -o "${TEMP_REQUIREMENTS}" --without-hashes +# We need to remove local packages from the requirements file. +sed -i '/^acme @/d; /certbot/d;' "${TEMP_REQUIREMENTS}" +# Poetry currently will not include pip, setuptools, or wheel in lockfiles or +# requirements files. This was resolved by +# https://github.com/python-poetry/poetry/pull/2826, but as of writing this it +# hasn't been included in a release yet. For now, we continue to keep +# pipstrap's pinning separate which has the added benefit of having it continue +# to check hashes when pipstrap is run directly. +"${STRIP_HASHES}" "${PIPSTRAP_CONSTRAINTS}" >> "${TEMP_REQUIREMENTS}" + +cat << EOF > "$REQUIREMENTS_FILE" +# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using +# that script. +# +# It is normally used as constraints to pip, however, it has the name +# requirements.txt so that is scanned by GitHub. See +# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems +# for more info. +EOF +cat "${TEMP_REQUIREMENTS}" >> "${REQUIREMENTS_FILE}" diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml new file mode 100644 index 000000000..4dee88d51 --- /dev/null +++ b/tools/pinning/pyproject.toml @@ -0,0 +1,90 @@ +[tool.poetry] +name = "certbot-pinner" +version = "0.1.0" +description = "A simple project for pinning Certbot's dependencies using Poetry." +authors = ["Certbot Project"] +license = "Apache License 2.0" + +[tool.poetry.dependencies] +python = "^3.6" + +# Local dependencies +# Any local packages that have dependencies on other local packages must be +# listed below before the package it depends on. For instance, certbot depends +# on acme so certbot must be listed before acme. +certbot-ci = {path = "../../certbot-ci", extras = ["docs"]} +certbot-compatibility-test = {path = "../../certbot-compatibility-test", extras = ["docs"]} +certbot-dns-cloudflare = {path = "../../certbot-dns-cloudflare", extras = ["docs"]} +certbot-dns-cloudxns = {path = "../../certbot-dns-cloudxns", extras = ["docs"]} +certbot-dns-digitalocean = {path = "../../certbot-dns-digitalocean", extras = ["docs"]} +certbot-dns-dnsimple = {path = "../../certbot-dns-dnsimple", extras = ["docs"]} +certbot-dns-dnsmadeeasy = {path = "../../certbot-dns-dnsmadeeasy", extras = ["docs"]} +certbot-dns-gehirn = {path = "../../certbot-dns-gehirn", extras = ["docs"]} +certbot-dns-google = {path = "../../certbot-dns-google", extras = ["docs"]} +certbot-dns-linode = {path = "../../certbot-dns-linode", extras = ["docs"]} +certbot-dns-luadns = {path = "../../certbot-dns-luadns", extras = ["docs"]} +certbot-dns-nsone = {path = "../../certbot-dns-nsone", extras = ["docs"]} +certbot-dns-ovh = {path = "../../certbot-dns-ovh", extras = ["docs"]} +certbot-dns-rfc2136 = {path = "../../certbot-dns-rfc2136", extras = ["docs"]} +certbot-dns-route53 = {path = "../../certbot-dns-route53", extras = ["docs"]} +certbot-dns-sakuracloud = {path = "../../certbot-dns-sakuracloud", extras = ["docs"]} +certbot-nginx = {path = "../../certbot-nginx", extras = ["docs"]} +certbot-apache = {path = "../../certbot-apache", extras = ["dev"]} +certbot = {path = "../../certbot", extras = ["dev", "docs"]} +acme = {path = "../../acme", extras = ["dev", "docs"]} +letstest = {path = "../../letstest"} +windows-installer = {path = "../../windows-installer"} + +# Extra dependencies +# awscli is just listed here as a performance optimization. As of writing this, +# there are some conflicts in shared dependencies between it and other packages +# we depend on. To try and resolve them, poetry searches through older versions +# of awscli to see if that resolves the conflict, but there are over 1000 +# versions of awscli on PyPI so this process takes too long. Providing a high +# minimum version here prevents poetry from searching this path and it fairly +# quickly resolves the dependency conflict in another way. +awscli = ">=1.19.62" +# As of writing this, cython is a build dependency of pyyaml. Since there +# doesn't appear to be a good way to automatically track down and pin build +# dependencies in Python (see +# https://discuss.python.org/t/how-to-pin-build-dependencies/8238), we list it +# as a dependency here to ensure a version of cython is pinned for extra +# stability. +cython = "*" +# We install mock in our "external-mock" tox environment to test that we didn't +# break Certbot's test API which used to always use mock objects from the 3rd +# party mock library. We list the mock dependency here so that is pinned, but +# we don't depend on it in Certbot to avoid installing mock when it's not +# needed. This dependency can be removed here once Certbot's support for the +# 3rd party mock library has been dropped. +mock = "*" +# Upgrading coverage, pylint, pytest, and some of pytest's plugins causes many +# test failures so let's pin these packages back for now. +coverage = "4.5.4" +pylint = "2.4.3" +pytest = "3.2.5" +pytest-forked = "0.2" +# We were originally pinning back python-augeas for certbot-auto because we +# found the way older versions of the library linked to Augeas were more +# reliable. That's no longer a concern, however, we continue to pin back the +# library for now because it causes Certbot tests on Windows to fail. See +# https://github.com/certbot/certbot/issues/8732. +python-augeas = "0.5.0" +# Because some parts of Certbot documentation are generated from zope.interfaces, +# we need the Sphinx plugin repoze.sphinx.autointerface. This plugin is not +# currently compatible with Sphinx>=4 though so we're pinning an older version +# for now. See https://github.com/repoze/repoze.sphinx.autointerface/issues/16 +# for more info. This pinning could also be removed by the removal of +# zope.interface in Certbot. +sphinx = "<4" +# Library traitlets is a transitive dependency of ipdb (traitlets -> ipython -> ipdb). +# Version 5.x is incompatible with Python 3.6 but for some reasons, poetry fails to +# add the appropriate marker and allows this version to be installed under Python 3.6. +# We add a pinning to not create a set of requirements incompatible with Python 3.6. +traitlets = "<5" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/tools/pip_install.py b/tools/pip_install.py index c1c81482b..054ca1229 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -3,7 +3,7 @@ # to 1, a combination of tools/oldest_constraints.txt, # tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the # top level of the package's directory is used, otherwise, a combination of -# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# tools/certbot_constraints.txt and tools/dev_constraints.txt is used. The # other file always takes precedence over tools/dev_constraints.txt. If # CERTBOT_OLDEST is set, this script must be run with `-e ` and # no other arguments. @@ -39,52 +39,34 @@ def find_tools_path(): return os.path.dirname(readlink.main(__file__)) -def certbot_oldest_processing(tools_path, args, test_constraints): +def certbot_oldest_processing(tools_path, args, constraints_path): if args[0] != '-e' or len(args) != 2: raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' 'with a single -e argument.') # remove any extras such as [dev] pkg_dir = re.sub(r'\[\w+\]', '', args[1]) + # The order of the files in this list matters as files specified later can + # override the pinnings found in earlier files. + pinning_files = [os.path.join(tools_path, 'dev_constraints.txt'), + os.path.join(tools_path, 'oldest_constraints.txt')] requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') - shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) # packages like acme don't have any local oldest requirements - if not os.path.isfile(requirements): - return None - + if os.path.isfile(requirements): + # We add requirements to the end of the list so it can override + # anything that it needs to. + pinning_files.append(requirements) + else: + requirements = None + with open(constraints_path, 'w') as fd: + fd.write(merge_module.main(*pinning_files)) return requirements -def certbot_normal_processing(tools_path, test_constraints): +def certbot_normal_processing(tools_path, constraints_path): repo_path = os.path.dirname(tools_path) - certbot_requirements = os.path.normpath(os.path.join( - repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) - with open(certbot_requirements, 'r') as fd: - certbot_reqs = fd.readlines() - with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd: - pipstrap_reqs = fd.readlines() - with open(test_constraints, 'w') as fd: - data_certbot = "\n".join(strip_hashes.process_entries(certbot_reqs)) - data_pipstrap = "\n".join(strip_hashes.process_entries(pipstrap_reqs)) - data = "\n".join([data_certbot, data_pipstrap]) - fd.write(data) - - -def merge_requirements(tools_path, requirements, test_constraints, all_constraints): - # Order of the files in the merge function matters. - # Indeed version retained for a given package will be the last version - # found when following all requirements in the given order. - # Here is the order by increasing priority: - # 1) The general development constraints (tools/dev_constraints.txt) - # 2) The general tests constraints (oldest_requirements.txt or - # certbot-auto's dependency-requirements.txt + pipstrap's constraints - # for the normal processing) - # 3) The local requirement file, typically local-oldest-requirement in oldest tests - files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] - if requirements: - files.append(requirements) - merged_requirements = merge_module.main(*files) - with open(all_constraints, 'w') as fd: - fd.write(merged_requirements) + requirements = os.path.normpath(os.path.join( + repo_path, 'tools/requirements.txt')) + shutil.copy(requirements, constraints_path) def call_with_print(command, env=None): @@ -105,24 +87,21 @@ def main(args): tools_path = find_tools_path() with temporary_directory() as working_dir: - test_constraints = os.path.join(working_dir, 'test_constraints.txt') - all_constraints = os.path.join(working_dir, 'all_constraints.txt') - if os.environ.get('CERTBOT_NO_PIN') == '1': # With unpinned dependencies, there is no constraint pip_install_with_print(' '.join(args)) else: # Otherwise, we merge requirements to build the constraints and pin dependencies + constraints_path = os.path.join(working_dir, 'constraints.txt') requirements = None if os.environ.get('CERTBOT_OLDEST') == '1': - requirements = certbot_oldest_processing(tools_path, args, test_constraints) + requirements = certbot_oldest_processing(tools_path, args, constraints_path) else: - certbot_normal_processing(tools_path, test_constraints) + certbot_normal_processing(tools_path, constraints_path) env = os.environ.copy() - env["PIP_CONSTRAINT"] = all_constraints + env["PIP_CONSTRAINT"] = constraints_path - merge_requirements(tools_path, requirements, test_constraints, all_constraints) if requirements: # This branch is executed during the oldest tests # First step, install the transitive dependencies of oldest requirements # in respect with oldest constraints. @@ -134,6 +113,7 @@ def main(args): pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"' .format(requirements)) + print(' '.join(args)) pip_install_with_print(' '.join(args), env=env) diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index abfe9f214..646344f3a 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -1,14 +1,11 @@ #!/usr/bin/env python -# pip installs packages in editable mode using certbot-auto's requirements file +# pip installs packages in editable mode using tools/certbot_constraints.txt # as constraints # # cryptography is currently using this script in their CI at # https://github.com/pyca/cryptography/blob/a02fdd60d98273ca34427235c4ca96687a12b239/.travis/downstream.d/certbot.sh#L8-L9. # We should try to remember to keep their repo updated if we make any changes # to this script which may break things for them. - -from __future__ import absolute_import - import sys import pip_install diff --git a/tools/pipstrap.py b/tools/pipstrap.py index e6b746916..2b2e3dcbb 100755 --- a/tools/pipstrap.py +++ b/tools/pipstrap.py @@ -1,11 +1,9 @@ #!/usr/bin/env python """Uses pip to upgrade Python packaging tools to pinned versions.""" -from __future__ import absolute_import import os import pip_install - _REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_constraints.txt") diff --git a/tools/pipstrap_constraints.txt b/tools/pipstrap_constraints.txt index 5de9e147d..54ab8b429 100644 --- a/tools/pipstrap_constraints.txt +++ b/tools/pipstrap_constraints.txt @@ -10,9 +10,9 @@ pip==20.2.4 \ --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \ --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1 -setuptools==44.1.1 \ - --hash=sha256:27a714c09253134e60a6fa68130f78c7037e5562c4f21f8f318f2ae900d152d5 \ - --hash=sha256:c67aa55db532a0dadc4d2e20ba9961cbd3ccc84d544e9029699822542b5a476b +setuptools==54.1.2 \ + --hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b \ + --hash=sha256:ebd0148faf627b569c8d2a1b20f5d3b09c873f12739d71c7ee88f037d5be82ff wheel==0.35.1 \ --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \ --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f diff --git a/tools/readlink.py b/tools/readlink.py index 446c8ebdc..c1e0c2e61 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -6,8 +6,6 @@ useful as there are often differences in readlink on different platforms. """ -from __future__ import print_function - import os import sys diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 000000000..9e442f411 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,186 @@ +# This file was generated by tools/pinning/pin.sh and can be updated using +# that script. +# +# It is normally used as constraints to pip, however, it has the name +# requirements.txt so that is scanned by GitHub. See +# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems +# for more info. +alabaster==0.7.12; python_version >= "3.6" +apacheconfig==0.3.2; python_version >= "3.6" +apipkg==1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +appnope==0.1.2 +astroid==2.3.3; python_version >= "3.6" +attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +awscli==1.19.69; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") +azure-devops==6.0.0b4; python_version >= "3.6" +babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +backcall==0.2.0 +bcrypt==3.2.0; python_version >= "3.6" +beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" +bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +boto3==1.17.69; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.69; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +cachy==0.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +cleo==0.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +clikit==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +cloudflare==2.8.15; python_version >= "3.6" +colorama==0.4.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +configargparse==1.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +configobj==5.0.6; python_version >= "3.6" +coverage==4.5.4; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0" and python_version < "4") +crashtest==0.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +cryptography==3.4.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +cython==0.29.23; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") +decorator==5.0.7 +deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +distlib==0.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +dns-lexicon==3.6.0; python_version >= "3.6" and python_version < "4.0" +dnspython==2.1.0; python_version >= "3.6" +docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docutils==0.15.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") +execnet==1.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +fabric==2.6.0; python_version >= "3.6" +filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0" +google-api-core==1.26.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +google-api-python-client==2.3.0; python_version >= "3.6" +google-auth-httplib2==0.1.0; python_version >= "3.6" +google-auth==1.30.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +httplib2==0.19.1; python_version >= "3.6" +idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" +imagesize==1.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +importlib-resources==5.1.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" +invoke==1.5.0; python_version >= "3.6" +ipdb==0.13.7; python_version >= "3.6" +ipython-genutils==0.2.0; python_version == "3.6" +ipython==7.16.1; python_version == "3.6" +ipython==7.23.1; python_version >= "3.7" +isodate==0.6.0; python_version >= "3.6" +isort==4.3.21; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +jedi==0.18.0 +jeepney==0.6.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +jinja2==2.11.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +josepy==1.8.0; python_version >= "3.6" +jsonlines==2.0.0; python_version >= "3.6" +jsonpickle==2.0.0; python_version >= "3.6" +jsonschema==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +keyring==21.8.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +lazy-object-proxy==1.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +lockfile==0.12.2 +markupsafe==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +matplotlib-inline==0.1.2; python_version >= "3.7" +mccabe==0.6.1; python_version >= "3.6" +mock==4.0.3; python_version >= "3.6" +msgpack==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +msrest==0.6.21; python_version >= "3.6" +mypy-extensions==0.4.3; python_version >= "3.6" +mypy==0.812; python_version >= "3.6" +oauth2client==4.1.3; python_version >= "3.6" +oauthlib==3.1.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" +paramiko==2.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +parsedatetime==2.6; python_version >= "3.6" +parso==0.8.2; python_version == "3.6" +pastel==0.2.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pathlib2==2.3.5; python_version >= "3.6" +pexpect==4.8.0 +pickleshare==0.7.5 +pkginfo==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +ply==3.11; python_version >= "3.6" +poetry-core==1.0.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +poetry==1.1.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +prompt-toolkit==3.0.3 +protobuf==3.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pygithub==1.55; python_version >= "3.6" +pygments==2.9.0 +pyjwt==2.1.0; python_version >= "3.6" +pylev==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pylint==2.4.3; python_version >= "3.5" +pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +pynsist==2.7; python_version >= "3.6" +pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pypiwin32==223; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") +pyrfc3339==1.1; python_version >= "3.6" +pyrsistent==0.17.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest-cov==2.6.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest-forked==0.2 +pytest-xdist==1.24.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest==3.2.5 +python-augeas==0.5.0 +python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +python-digitalocean==1.16.0; python_version >= "3.6" +python-dotenv==0.17.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" +pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "win32" +pywin32==300; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") +pyyaml==5.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4.0" +readme-renderer==29.0; python_version >= "3.6" +repoze.sphinx.autointerface==0.8; python_version >= "3.6" +requests-download==0.1.2; python_version >= "3.6" +requests-file==1.5.1; python_version >= "3.6" and python_version < "4.0" +requests-oauthlib==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.6.0" and python_version < "4.0" +rfc3986==1.5.0; python_version >= "3.6" +rsa==4.7.2; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +s3transfer==0.4.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +six==1.16.0; python_version == "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version == "3.6" +snowballstemmer==2.1.0; python_version >= "3.6" +soupsieve==2.2.1; python_version >= "3.6" +sphinx-rtd-theme==0.5.2; python_version >= "3.6" +sphinx==3.5.4; python_version >= "3.5" +sphinxcontrib-applehelp==1.0.2; python_version >= "3.6" +sphinxcontrib-devhelp==1.0.2; python_version >= "3.6" +sphinxcontrib-htmlhelp==1.0.3; python_version >= "3.6" +sphinxcontrib-jsmath==1.0.1; python_version >= "3.6" +sphinxcontrib-qthelp==1.0.3; python_version >= "3.6" +sphinxcontrib-serializinghtml==1.1.4; python_version >= "3.6" +texttable==1.6.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +tldextract==3.1.0; python_version >= "3.6" and python_version < "4.0" +toml==0.10.2; python_version == "3.6" and python_full_version < "3.0.0" or python_version > "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.5.0" or python_version > "3.6" and python_full_version >= "3.5.0" +tomlkit==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +tox==3.23.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +tqdm==4.60.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +traitlets==4.3.3 +twine==3.3.0; python_version >= "3.6" +typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6" +typing-extensions==3.10.0.0; python_version >= "3.6" +uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +urllib3==1.26.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" +virtualenv==20.4.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +wcwidth==0.2.5; python_version == "3.6" +webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +websocket-client==0.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +wrapt==1.11.2; python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0") +yarg==0.1.9; python_version >= "3.6" +zipp==3.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" +zope.component==5.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.event==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.hookable==5.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.interface==5.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pip==20.2.4 +setuptools==54.1.2 +wheel==0.35.1 diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index e6a44240f..91854e9e0 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -1,33 +1,102 @@ #!/usr/bin/env python3 import argparse import datetime +import functools import glob +from multiprocessing import Manager +from multiprocessing import Pool +from multiprocessing import Process +from multiprocessing.managers import SyncManager +import os +from os.path import basename +from os.path import dirname +from os.path import exists +from os.path import join +from os.path import realpath import re +import shutil import subprocess import sys +import tempfile +from threading import Lock import time -from multiprocessing import Pool, Process, Manager -from os.path import join, realpath, dirname, basename, exists +from typing import Dict +from typing import List +from typing import Set +from typing import Tuple CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] -def _execute_build(target, archs, status, workspace): - process = subprocess.Popen([ - 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', - '--build-on', ','.join(archs) - ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, cwd=workspace) - - process_output = [] - for line in process.stdout: - process_output.append(line) - _extract_state(target, line, status) - - return process.wait(), process_output +# In Python, stdout and stderr are buffered in each process by default. When +# printing output from multiple processes, this can cause delays in printing +# output with lines from different processes being interleaved depending +# on when the output for that process is flushed. To prevent this, we override +# print so that it always flushes its output. Disabling output buffering can +# also be done through command line flags or environment variables set when the +# Python process starts, but this approach was taken instead to ensure +# consistent behavior regardless of how the script is invoked. +print = functools.partial(print, flush=True) -def _build_snap(target, archs, status, running, lock): +def _snap_log_name(target: str, arch: str): + return f'{target}_{arch}.txt' + + +def _execute_build( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + workspace: str) -> Tuple[int, List[str]]: + + # Snapcraft remote-build has a recover feature, that will make it reconnect to an existing + # build on Launchpad if possible. However, the signature used to retrieve a potential + # build is not based on the content of the sources used to build a snap, but on a hash + # of the snapcraft current working directory (the path itself, not the content). + # It means that every build started from /my/path/to/certbot will always be considered + # as the same build, whatever the actual sources are. + # To circumvent this, we create a temporary folder and use it as a workspace to build + # the snap: this path is random, making the recover feature effectively noop. + with tempfile.TemporaryDirectory() as temp_workspace: + ignore = None + if target == 'certbot': + ignore = shutil.ignore_patterns(".git", "venv*", ".tox") + shutil.copytree(workspace, temp_workspace, + dirs_exist_ok=True, symlinks=True, ignore=ignore) # type:ignore + + with tempfile.TemporaryDirectory() as tempdir: + environ = os.environ.copy() + environ['XDG_CACHE_HOME'] = tempdir + process = subprocess.Popen([ + 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', + '--build-on', ','.join(archs)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True, env=environ, cwd=temp_workspace) + + process_output: List[str] = [] + for line in process.stdout: + process_output.append(line) + _extract_state(target, line, status) + + if any(state for state in status[target].values() if state == 'Chroot problem'): + # On this error the snapcraft process stales. Let's finish it. + process.kill() + + process_state = process.wait() + + for path in glob.glob(join(temp_workspace, '*.snap')): + shutil.copy(path, workspace) + for arch in archs: + log_name = _snap_log_name(target, arch) + log_path = join(temp_workspace, log_name) + if exists(log_path): + shutil.copy(log_path, workspace) + + return process_state, process_output + + +def _build_snap( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + running: Dict[str, bool], output_lock: Lock) -> bool: status[target] = {arch: '...' for arch in archs} if target == 'certbot': @@ -35,17 +104,19 @@ def _build_snap(target, archs, status, running, lock): else: workspace = join(CERTBOT_DIR, target) + build_success = False retry = 3 while retry: exit_code, process_output = _execute_build(target, archs, status, workspace) + with output_lock: + print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with ' + f'exit code {exit_code}.') - print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with ' - f'exit code {exit_code}.') - sys.stdout.flush() - - with lock: - dump_output = exit_code != 0 - failed_archs = [arch for arch in archs if status[target][arch] == 'Failed to build'] + failed_archs = [arch for arch in archs if status[target][arch] != 'Successfully built'] + # If the command failed or any architecture wasn't built + # successfully, let's try to print all the output about the problem + # that we can. + dump_output = exit_code != 0 or failed_archs if exit_code == 0 and not failed_archs: # We expect to have all target snaps available, or something bad happened. snaps_list = glob.glob(join(workspace, '*.snap')) @@ -54,16 +125,12 @@ def _build_snap(target, archs, status, running, lock): f'(current list: {snaps_list}).') dump_output = True else: + build_success = True break - if failed_archs: - # We expect each failed build to have a log file, or something bad happened. - for arch in failed_archs: - if not exists(join(workspace, f'{target}_{arch}.txt')): - dump_output = True - print(f'Missing output on a failed build {target} for {arch}.') if dump_output: print(f'Dumping snapcraft remote-build output build for {target}:') print('\n'.join(process_output)) + _dump_failed_build_logs(target, archs, status, workspace) # Retry the remote build if it has been interrupted (non zero status code) # or if some builds have failed. @@ -71,25 +138,30 @@ def _build_snap(target, archs, status, running, lock): running[target] = False - return {target: workspace} + return build_success -def _extract_state(project, output, status): +def _extract_state(project: str, output: str, status: Dict[str, Dict[str, str]]) -> None: + state = status[project] + + if "Sending build data to Launchpad..." in output: + for arch in state.keys(): + state[arch] = "Sending build data" + match = re.match(r'^.*arch=(\w+)\s+state=([\w ]+).*$', output) if match: arch = match.group(1) - state = status[project] state[arch] = match.group(2) - # You need to reassign the value of status[project] here (rather than doing - # something like status[project][arch] = match.group(2)) for the state change - # to propagate to other processes. See - # https://docs.python.org/3.8/library/multiprocessing.html#proxy-objects for - # more info. - status[project] = state + # You need to reassign the value of status[project] here (rather than doing + # something like status[project][arch] = match.group(2)) for the state change + # to propagate to other processes. See + # https://docs.python.org/3.8/library/multiprocessing.html#proxy-objects for + # more info. + status[project] = state -def _dump_status_helper(archs, status): +def _dump_status_helper(archs: Set[str], status: Dict[str, Dict[str, str]]) -> None: headers = ['project', *archs] print(''.join(f'| {item:<25}' for item in headers)) print(f'|{"-" * 26}' * len(headers)) @@ -98,56 +170,52 @@ def _dump_status_helper(archs, status): print(f'|{"-" * 26}' * len(headers)) print() - sys.stdout.flush() - -def _dump_status(archs, status, running): +def _dump_status( + archs: Set[str], status: Dict[str, Dict[str, str]], + running: Dict[str, bool], output_lock: Lock) -> None: while any(running.values()): - print(f'Remote build status at {datetime.datetime.now()}') - _dump_status_helper(archs, status) + with output_lock: + print(f'Remote build status at {datetime.datetime.now()}') + _dump_status_helper(archs, status) time.sleep(10) -def _dump_results(targets, archs, status, workspaces): - failures = False - for target in targets: - for arch in archs: - result = status[target][arch] +def _dump_failed_build_logs( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + workspace: str) -> None: + for arch in archs: + result = status[target][arch] - if result != 'Successfully built': - failures = True + if result != 'Successfully built': + failures = True - build_output_path = join(workspaces[target], f'{target}_{arch}.txt') - if not exists(build_output_path): - build_output = f'No output has been dumped by snapcraft remote-build.' - else: - with open(join(workspaces[target], f'{target}_{arch}.txt')) as file_h: - build_output = file_h.read() + build_output_name = _snap_log_name(target, arch) + build_output_path = join(workspace, build_output_name) + if not exists(build_output_path): + build_output = f'No output has been dumped by snapcraft remote-build.' + else: + with open(build_output_path) as file_h: + build_output = file_h.read() - print(f'Output for failed build target={target} arch={arch}') - print('-------------------------------------------') - print(build_output) - print('-------------------------------------------') - print() + print(f'Output for failed build target={target} arch={arch}') + print('-------------------------------------------') + print(build_output) + print('-------------------------------------------') + print() - if not failures: - print('All builds succeeded.') - else: - print('Some builds failed.') - print() +def _dump_results(archs: Set[str], status: Dict[str, Dict[str, str]]) -> None: print(f'Results for remote build finished at {datetime.datetime.now()}') _dump_status_helper(archs, status) - return failures - def main(): parser = argparse.ArgumentParser() parser.add_argument('targets', nargs='+', choices=['ALL', 'DNS_PLUGINS', 'certbot', *PLUGINS], help='the list of snaps to build') - parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default=['amd64'], - help='the architectures for which snaps are built') + parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], + default=['amd64'], help='the architectures for which snaps are built') parser.add_argument('--timeout', type=int, default=None, help='build process will fail after the provided timeout (in seconds)') args = parser.parse_args() @@ -174,15 +242,19 @@ def main(): print(f' - projects: {", ".join(sorted(targets))}') print() - with Manager() as manager, Pool(processes=len(targets)) as pool: - status = manager.dict() + manager: SyncManager = Manager() + pool = Pool(processes=len(targets)) + with manager, pool: + status: Dict[str, Dict[str, str]] = manager.dict() running = manager.dict({target: True for target in targets}) - lock = manager.Lock() + # While multiple processes are running, this lock should be acquired + # before printing output. + output_lock = manager.Lock() - async_results = [pool.apply_async(_build_snap, (target, archs, status, running, lock)) + async_results = [pool.apply_async(_build_snap, (target, archs, status, running, output_lock)) for target in targets] - process = Process(target=_dump_status, args=(archs, status, running)) + process = Process(target=_dump_status, args=(archs, status, running, output_lock)) process.start() try: @@ -191,11 +263,16 @@ def main(): if process.is_alive(): raise ValueError(f"Timeout out reached ({args.timeout} seconds) during the build!") - workspaces = {} + build_success = True for async_result in async_results: - workspaces.update(async_result.get()) + if not async_result.get(): + build_success = False - if _dump_results(targets, archs, status, workspaces): + _dump_results(archs, status) + if build_success: + print('All builds succeeded.') + else: + print('Some builds failed.') raise ValueError("There were failures during the build!") finally: process.terminate() diff --git a/tools/snap/generate_dnsplugins_all.sh b/tools/snap/generate_dnsplugins_all.sh index 40404bf9b..769b4ef4d 100755 --- a/tools/snap/generate_dnsplugins_all.sh +++ b/tools/snap/generate_dnsplugins_all.sh @@ -9,8 +9,5 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_snapcraft.sh $PLUGIN_PATH bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH # Create constraints file - "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt) \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \ - > "${PLUGIN_PATH}"/snap-constraints.txt + cp "${CERTBOT_DIR}"/tools/requirements.txt "${PLUGIN_PATH}"/snap-constraints.txt done diff --git a/tools/snap/generate_dnsplugins_snapcraft.sh b/tools/snap/generate_dnsplugins_snapcraft.sh index d93d8ec73..139c8b0d3 100755 --- a/tools/snap/generate_dnsplugins_snapcraft.sh +++ b/tools/snap/generate_dnsplugins_snapcraft.sh @@ -35,7 +35,14 @@ parts: - PIP_CONSTRAINT: \$SNAPCRAFT_PART_SRC/snap-constraints.txt - SNAP_BUILD: "True" # To build cryptography and cffi if needed - build-packages: [gcc, libffi-dev, libssl-dev, python3-dev] + build-packages: + - gcc + - git + - build-essential + - libssl-dev + - libffi-dev + - python3-dev + - cargo certbot-metadata: plugin: dump source: . diff --git a/tools/venv.py b/tools/venv.py index 9f7488008..f3f5781fa 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -44,8 +44,12 @@ REQUIREMENTS = [ '-e certbot-nginx', '-e certbot-compatibility-test', '-e certbot-ci', + '-e letstest', ] +if sys.platform == 'win32': + REQUIREMENTS.append('-e windows-installer') + VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') diff --git a/tox.ini b/tox.ini index 9f63b897c..a45766d87 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py3-cover,lint,mypy +envlist = py3-cover,lint,mypy [base] # pip installs the requested packages in editable mode @@ -151,6 +151,7 @@ commands = {[base]install_packages} python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} +// TODO: Re-enable strict checks for optionals with appropriate type corrections or code redesign. [testenv:mypy] basepython = python3 commands = @@ -181,12 +182,9 @@ commands = {[base]pip_install} acme certbot certbot-apache certbot-nginx python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata -# This is a duplication of the command line in testenv:le_auto to -# allow users to run the modification check by running `tox` [testenv:modification] commands = python {toxinidir}/tests/modification-check.py -passenv = TARGET_BRANCH [testenv:apache_compat] commands = @@ -279,36 +277,26 @@ passenv = DOCKER_* setenv = {[testenv:oldest]setenv} [testenv:test-farm-tests-base] -changedir = tests/letstest -deps = -rtests/letstest/requirements.txt +changedir = letstest +# The package to install is in the current working directory because of the +# value of changedir. +commands = {[base]pip_install} . passenv = AWS_* setenv = AWS_DEFAULT_REGION=us-east-1 [testenv:test-farm-apache2] changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} -passenv = {[testenv:test-farm-tests-base]passenv} -setenv = {[testenv:test-farm-tests-base]setenv} - -[testenv:test-farm-leauto-upgrades] -changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_leauto_upgrades.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} -passenv = {[testenv:test-farm-tests-base]passenv} -setenv = {[testenv:test-farm-tests-base]setenv} - -[testenv:test-farm-certonly-standalone] -changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_letsencrypt_auto_certonly_standalone.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} +commands = + {[testenv:test-farm-tests-base]commands} + {toxinidir}/tools/retry.sh letstest targets/apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir} passenv = {[testenv:test-farm-tests-base]passenv} setenv = {[testenv:test-farm-tests-base]setenv} [testenv:test-farm-sdists] changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} +commands = + {[testenv:test-farm-tests-base]commands} + {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir} passenv = {[testenv:test-farm-tests-base]passenv} setenv = {[testenv:test-farm-tests-base]setenv} diff --git a/windows-installer/certbot.ico b/windows-installer/assets/certbot.ico similarity index 100% rename from windows-installer/certbot.ico rename to windows-installer/assets/certbot.ico diff --git a/windows-installer/renew-down.ps1 b/windows-installer/assets/renew-down.ps1 similarity index 100% rename from windows-installer/renew-down.ps1 rename to windows-installer/assets/renew-down.ps1 diff --git a/windows-installer/renew-up.ps1 b/windows-installer/assets/renew-up.ps1 similarity index 100% rename from windows-installer/renew-up.ps1 rename to windows-installer/assets/renew-up.ps1 diff --git a/windows-installer/run.bat b/windows-installer/assets/run.bat similarity index 100% rename from windows-installer/run.bat rename to windows-installer/assets/run.bat diff --git a/windows-installer/template.nsi b/windows-installer/assets/template.nsi similarity index 99% rename from windows-installer/template.nsi rename to windows-installer/assets/template.nsi index 64bceb065..566e1b004 100644 --- a/windows-installer/template.nsi +++ b/windows-installer/assets/template.nsi @@ -1,7 +1,7 @@ -; This NSIS template is based on the built-in one in pynsist 2.6. +; This NSIS template is based on the built-in one in pynsist 2.7. ; Added lines are enclosed within "CERTBOT CUSTOM BEGIN/END" comments. ; If pynsist is upgraded, this template must be updated if necessary using the new built-in one. -; Original file can be found here: https://github.com/takluyver/pynsist/blob/2.6/nsist/pyapp.nsi +; Original file can be found here: https://github.com/takluyver/pynsist/blob/2.7/nsist/pyapp.nsi !define PRODUCT_NAME "[[ib.appname]]" !define PRODUCT_VERSION "[[ib.version]]" diff --git a/windows-installer/setup.py b/windows-installer/setup.py new file mode 100644 index 000000000..cddc9ea18 --- /dev/null +++ b/windows-installer/setup.py @@ -0,0 +1,45 @@ +from setuptools import find_packages +from setuptools import setup + +version = '1.0' + +setup( + name='windows-installer', + version=version, + description='Environment to build the Certbot Windows installer', + url='https://github.com/letsencrypt/letsencrypt', + author="Certbot Project", + author_email='certbot-dev@eff.org', + license='Apache License 2.0', + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Operating System :: Microsoft :: Windows', + 'Topic :: Software Development :: Build Tools', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=[ + # pynsist is pinned to an exact version so we can update + # assets/template.nsi as needed. The file is based on the default + # pynsist NSIS template and pynsist's documentation warns that custom + # templates may need to be updated for them to work with new versions + # of pynsist. See + # https://pynsist.readthedocs.io/en/latest/cfgfile.html#build-section. + 'pynsist==2.7' + ], + entry_points={ + 'console_scripts': [ + 'construct-windows-installer = windows_installer.construct:main', + ], + }, +) diff --git a/windows-installer/windows_installer/__init__.py b/windows-installer/windows_installer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/windows-installer/construct.py b/windows-installer/windows_installer/construct.py similarity index 65% rename from windows-installer/construct.py rename to windows-installer/windows_installer/construct.py index 0684b3c25..91df6b714 100644 --- a/windows-installer/construct.py +++ b/windows-installer/windows_installer/construct.py @@ -1,21 +1,33 @@ #!/usr/bin/env python3 -import contextlib import ctypes import os import shutil import struct import subprocess import sys -import tempfile import time -PYTHON_VERSION = (3, 8, 6) +PYTHON_VERSION = (3, 8, 9) PYTHON_BITNESS = 32 -PYWIN32_VERSION = 300 # do not forget to edit pywin32 dependency accordingly in setup.py NSIS_VERSION = '3.06.1' def main(): + if os.name != 'nt': + raise RuntimeError('This script must be run under Windows.') + + if ctypes.windll.shell32.IsUserAnAdmin() == 0: + # Administrator privileges are required to properly install NSIS through Chocolatey + raise RuntimeError('This script must be run with administrator privileges.') + + if sys.version_info[:2] != PYTHON_VERSION[:2]: + raise RuntimeError('This script must be run with Python {0}' + .format('.'.join(str(item) for item in PYTHON_VERSION[0:2]))) + + if struct.calcsize('P') * 8 != PYTHON_BITNESS: + raise RuntimeError('This script must be run with a {0} bit version of Python.' + .format(PYTHON_BITNESS)) + build_path, repo_path, venv_path, venv_python = _prepare_environment() _copy_assets(build_path, repo_path) @@ -24,14 +36,14 @@ def main(): _prepare_build_tools(venv_path, venv_python, repo_path) _compile_wheels(repo_path, build_path, venv_python) - _build_installer(installer_cfg_path, venv_path) + _build_installer(installer_cfg_path) print('Done') -def _build_installer(installer_cfg_path, venv_path): +def _build_installer(installer_cfg_path): print('Build the installer') - subprocess.check_call([os.path.join(venv_path, 'Scripts', 'pynsist.exe'), installer_cfg_path]) + subprocess.check_call([sys.executable, '-m', 'nsist', installer_cfg_path]) def _compile_wheels(repo_path, build_path, venv_python): @@ -45,54 +57,31 @@ def _compile_wheels(repo_path, build_path, venv_python): # certbot_packages.extend([name for name in os.listdir(repo_path) if name.startswith('certbot-dns-')]) wheels_project = [os.path.join(repo_path, package) for package in certbot_packages] - with _prepare_constraints(repo_path) as constraints_file_path: - env = os.environ.copy() - env['PIP_CONSTRAINT'] = constraints_file_path - command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path] - command.extend(wheels_project) - subprocess.check_call(command, env=env) + constraints_file_path = os.path.join(repo_path, 'tools', 'requirements.txt') + env = os.environ.copy() + env['PIP_CONSTRAINT'] = constraints_file_path + command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path] + command.extend(wheels_project) + subprocess.check_call(command, env=env) def _prepare_build_tools(venv_path, venv_python, repo_path): print('Prepare build tools') subprocess.check_call([sys.executable, '-m', 'venv', venv_path]) subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pipstrap.py')]) - subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pip_install.py'), 'pynsist']) subprocess.check_call(['choco', 'upgrade', '--allow-downgrade', '-y', 'nsis', '--version', NSIS_VERSION]) -@contextlib.contextmanager -def _prepare_constraints(repo_path): - reqs_certbot = os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'dependency-requirements.txt') - reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt') - constraints_certbot = subprocess.check_output( - [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot], - universal_newlines=True) - constraints_pipstrap = subprocess.check_output( - [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_pipstrap], - universal_newlines=True) - workdir = tempfile.mkdtemp() - try: - constraints_file_path = os.path.join(workdir, 'constraints.txt') - with open(constraints_file_path, 'a') as file_h: - file_h.write(constraints_pipstrap) - file_h.write(constraints_certbot) - file_h.write('pywin32=={0}'.format(PYWIN32_VERSION)) - yield constraints_file_path - finally: - shutil.rmtree(workdir) - - def _copy_assets(build_path, repo_path): print('Copy assets') if os.path.exists(build_path): os.rename(build_path, '{0}.{1}.bak'.format(build_path, int(time.time()))) os.makedirs(build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'certbot.ico'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'run.bat'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'template.nsi'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'renew-up.ps1'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'renew-down.ps1'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'certbot.ico'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'run.bat'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'template.nsi'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'renew-up.ps1'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'renew-down.ps1'), build_path) def _generate_pynsist_config(repo_path, build_path): @@ -148,7 +137,7 @@ def _prepare_environment(): raise RuntimeError('Error: Chocolatey (https://chocolatey.org/) needs ' 'to be installed to run this script.') script_path = os.path.realpath(__file__) - repo_path = os.path.dirname(os.path.dirname(script_path)) + repo_path = os.path.dirname(os.path.dirname(os.path.dirname(script_path))) build_path = os.path.join(repo_path, 'windows-installer', 'build') venv_path = os.path.join(build_path, 'venv-config') venv_python = os.path.join(venv_path, 'Scripts', 'python.exe') @@ -157,18 +146,4 @@ def _prepare_environment(): if __name__ == '__main__': - if os.name != 'nt': - raise RuntimeError('This script must be run under Windows.') - - if ctypes.windll.shell32.IsUserAnAdmin() == 0: - # Administrator privileges are required to properly install NSIS through Chocolatey - raise RuntimeError('This script must be run with administrator privileges.') - - if sys.version_info[:2] != PYTHON_VERSION[:2]: - raise RuntimeError('This script must be run with Python {0}' - .format('.'.join(str(item) for item in PYTHON_VERSION[0:2]))) - - if struct.calcsize('P') * 8 != PYTHON_BITNESS: - raise RuntimeError('This script must be run with a {0} bit version of Python.' - .format(PYTHON_BITNESS)) main()