diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml index cffedfcb2..c8cbe7fb3 100644 --- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml @@ -85,12 +85,6 @@ jobs: IMAGE_NAME: macOS-10.15 PYTHON_VERSION: 3.8 TOXENV: test-farm-apache2 - farmtest-leauto-upgrades: - PYTHON_VERSION: 3.7 - TOXENV: test-farm-leauto-upgrades - farmtest-certonly-standalone: - PYTHON_VERSION: 3.7 - TOXENV: test-farm-certonly-standalone farmtest-sdists: PYTHON_VERSION: 3.7 TOXENV: test-farm-sdists diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index 28255919f..005b5ef48 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -12,6 +12,9 @@ jobs: DOCKER_ARCH: arm32v6 arm64v8: DOCKER_ARCH: arm64v8 + # The default timeout of 60 minutes is a little low for compiling + # cryptography on ARM architectures. + timeoutInMinutes: 180 steps: - bash: set -e && tools/docker/build.sh $(dockerTag) $DOCKER_ARCH displayName: Build the Docker images @@ -59,7 +62,13 @@ jobs: versionSpec: 3.8 architecture: x86 addToPath: true - - script: python windows-installer/construct.py + - script: | + python -m venv venv + venv\Scripts\python tools\pipstrap.py + venv\Scripts\python tools\pip_install.py -e windows-installer + displayName: Prepare Windows installer build environment + - script: | + venv\Scripts\construct-windows-installer displayName: Build Certbot installer - task: CopyFiles@2 inputs: @@ -116,13 +125,17 @@ jobs: - job: snaps_build pool: vmImage: ubuntu-18.04 + strategy: + matrix: + amd64: + SNAP_ARCH: amd64 + # Do not run the heavy non-amd64 builds for test branches + ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}: + armhf: + SNAP_ARCH: armhf + arm64: + SNAP_ARCH: arm64 timeoutInMinutes: 0 - variables: - # Do not run the heavy non-amd64 builds for test branches - ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}: - ARCHS: amd64 arm64 armhf - ${{ if startsWith(variables['Build.SourceBranchName'], 'test-') }}: - ARCHS: amd64 steps: - script: | set -e @@ -144,7 +157,7 @@ jobs: git config --global user.name "$(Build.RequestedFor)" mkdir -p ~/.local/share/snapcraft/provider/launchpad cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials - python3 tools/snap/build_remote.py ALL --archs ${ARCHS} --timeout 19800 + python3 tools/snap/build_remote.py ALL --archs ${SNAP_ARCH} --timeout 19800 displayName: Build snaps - script: | set -e @@ -154,7 +167,7 @@ jobs: - task: PublishPipelineArtifact@1 inputs: path: $(Build.ArtifactStagingDirectory) - artifact: snaps + artifact: snaps_$(SNAP_ARCH) displayName: Store snaps artifacts - job: snap_run dependsOn: snaps_build @@ -175,12 +188,12 @@ jobs: displayName: Install dependencies - task: DownloadPipelineArtifact@2 inputs: - artifact: snaps + artifact: snaps_amd64 path: $(Build.SourcesDirectory)/snap displayName: Retrieve Certbot snaps - script: | set -e - sudo snap install --dangerous --classic snap/certbot_*_amd64.snap + sudo snap install --dangerous --classic snap/certbot_*.snap displayName: Install Certbot snap - script: | set -e @@ -202,7 +215,7 @@ jobs: addToPath: true - task: DownloadPipelineArtifact@2 inputs: - artifact: snaps + artifact: snaps_amd64 path: $(Build.SourcesDirectory)/snap displayName: Retrieve Certbot snaps - script: | diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index fec11c6c5..c949af44a 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -40,13 +40,13 @@ jobs: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.9 TOXENV: py39-cover - linux-py37-lint: + linux-py39-lint: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 3.7 + PYTHON_VERSION: 3.9 TOXENV: lint - linux-py36-mypy: + linux-py39-mypy: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 3.6 + PYTHON_VERSION: 3.9 TOXENV: mypy linux-integration: IMAGE_NAME: ubuntu-18.04 @@ -56,6 +56,8 @@ jobs: apache-compat: IMAGE_NAME: ubuntu-18.04 TOXENV: apache_compat + # le-modification can be moved to the extended test suite once + # https://github.com/certbot/certbot/issues/8742 is resolved. le-modification: IMAGE_NAME: ubuntu-18.04 TOXENV: modification diff --git a/.azure-pipelines/templates/stages/deploy-stage.yml b/.azure-pipelines/templates/stages/deploy-stage.yml index ac2044f99..abbb1fd1a 100644 --- a/.azure-pipelines/templates/stages/deploy-stage.yml +++ b/.azure-pipelines/templates/stages/deploy-stage.yml @@ -37,6 +37,14 @@ stages: vmImage: ubuntu-18.04 variables: - group: certbot-common + strategy: + matrix: + amd64: + SNAP_ARCH: amd64 + arm32v6: + SNAP_ARCH: armhf + arm64v8: + SNAP_ARCH: arm64 steps: - bash: | set -e @@ -46,7 +54,7 @@ stages: displayName: Install dependencies - task: DownloadPipelineArtifact@2 inputs: - artifact: snaps + artifact: snaps_$(SNAP_ARCH) path: $(Build.SourcesDirectory)/snap displayName: Retrieve Certbot snaps - task: DownloadSecureFile@1 @@ -55,8 +63,7 @@ stages: secureFile: snapcraft.cfg - bash: | set -e - mkdir -p .snapcraft - ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg + snapcraft login --with $(snapcraftCfg.secureFilePath) for SNAP_FILE in snap/*.snap; do tools/retry.sh eval snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}" done diff --git a/.azure-pipelines/templates/steps/sphinx-steps.yml b/.azure-pipelines/templates/steps/sphinx-steps.yml index 23c258bbc..7e1d2ee9d 100644 --- a/.azure-pipelines/templates/steps/sphinx-steps.yml +++ b/.azure-pipelines/templates/steps/sphinx-steps.yml @@ -9,7 +9,7 @@ steps: do echo "" echo "##[group]Building $doc_path" - pip install -q -e $doc_path/..[docs] + tools/pip_install_editable.py $doc_path/..[docs] if ! sphinx-build -W --keep-going -b html $doc_path $doc_path/_build/html; then FINAL_STATUS=1 FAILED_BUILDS[${#FAILED_BUILDS[@]}]="${doc_path%/docs}" diff --git a/.azure-pipelines/templates/steps/tox-steps.yml b/.azure-pipelines/templates/steps/tox-steps.yml index ecf3d6032..3e5fb995d 100644 --- a/.azure-pipelines/templates/steps/tox-steps.yml +++ b/.azure-pipelines/templates/steps/tox-steps.yml @@ -1,6 +1,10 @@ steps: + # We run brew update because we've seen attempts to install an older version + # of a package fail. See + # https://github.com/actions/virtual-environments/issues/3165. - bash: | set -e + brew update brew install augeas condition: startswith(variables['IMAGE_NAME'], 'macOS') displayName: Install MacOS dependencies diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..f02dc2086 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: https://supporters.eff.org/donate/support-work-on-certbot diff --git a/ISSUE_TEMPLATE.md b/.github/issue_template.md similarity index 90% rename from ISSUE_TEMPLATE.md rename to .github/issue_template.md index 03917f8ca..87ecddc6c 100644 --- a/ISSUE_TEMPLATE.md +++ b/.github/issue_template.md @@ -7,7 +7,7 @@ questions. ## My operating system is (include version): -## I installed Certbot with (certbot-auto, OS package manager, pip, etc): +## I installed Certbot with (snap, OS package manager, pip, certbot-auto, etc): ## I ran this command and it produced this output: diff --git a/pull_request_template.md b/.github/pull_request_template.md similarity index 100% rename from pull_request_template.md rename to .github/pull_request_template.md diff --git a/.gitignore b/.gitignore index 5169defd6..285e68a42 100644 --- a/.gitignore +++ b/.gitignore @@ -4,13 +4,12 @@ build/ dist*/ /venv*/ -/kgs/ /.tox/ /releases*/ /log* letsencrypt.log certbot.log -letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64 +poetry.lock # coverage .coverage @@ -31,12 +30,6 @@ tags # auth --cert-path --chain-path /*.pem -# letstest -tests/letstest/letest-*/ -tests/letstest/*.pem -tests/letstest/venv/ -tests/letstest/venv3/ - .venv # pytest cache diff --git a/.isort.cfg b/.isort.cfg index 11c895f4d..6b17b459b 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,5 @@ [settings] skip_glob=venv* -skip=letsencrypt-auto-source force_sort_within_sections=True force_single_line=True order_by_type=False diff --git a/.pylintrc b/.pylintrc index a2468b0cf..e19077d8d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -8,7 +8,10 @@ jobs=0 # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). -#init-hook= +# CERTBOT COMMENT +# This is needed for pylint to import linter_plugin.py since +# https://github.com/PyCQA/pylint/pull/3396. +init-hook="import pylint.config, os, sys; sys.path.append(os.path.dirname(pylint.config.PYLINTRC))" # Profiled execution. profile=no diff --git a/Dockerfile-dev b/Dockerfile-dev index 86847f8fd..895dbdc0b 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,5 +1,5 @@ # This Dockerfile builds an image for development. -FROM debian:buster +FROM ubuntu:focal # Note: this only exposes the port to other docker containers. EXPOSE 80 443 @@ -8,8 +8,9 @@ WORKDIR /opt/certbot/src COPY . . RUN apt-get update && \ - apt-get install apache2 git python3-dev python3-venv gcc libaugeas0 \ - libssl-dev libffi-dev ca-certificates openssl nginx-light -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install apache2 git python3-dev \ + python3-venv gcc libaugeas0 libssl-dev libffi-dev ca-certificates \ + openssl nginx-light -y --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 82f7ae1b8..2c8190be5 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -5,6 +5,7 @@ import functools import hashlib import logging import socket +from typing import Type from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose @@ -29,7 +30,7 @@ class Challenge(jose.TypedJSONObjectWithFields): @classmethod def from_json(cls, jobj): try: - return super(Challenge, cls).from_json(jobj) + return super().from_json(jobj) except jose.UnrecognizedTypeError as error: logger.debug(error) return UnrecognizedChallenge.from_json(jobj) @@ -57,7 +58,7 @@ class UnrecognizedChallenge(Challenge): """ def __init__(self, jobj): - super(UnrecognizedChallenge, self).__init__() + super().__init__() object.__setattr__(self, "jobj", jobj) def to_partial_json(self): @@ -140,7 +141,7 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return True def to_partial_json(self): - jobj = super(KeyAuthorizationChallengeResponse, self).to_partial_json() + jobj = super().to_partial_json() jobj.pop('keyAuthorization', None) return jobj @@ -152,8 +153,8 @@ class KeyAuthorizationChallenge(_TokenChallenge, metaclass=abc.ABCMeta): that will be used to generate ``response``. :param str typ: type of the challenge """ - typ = NotImplemented - response_cls = NotImplemented + typ: str = NotImplemented + response_cls: Type[KeyAuthorizationChallengeResponse] = NotImplemented thumbprint_hash_function = ( KeyAuthorizationChallengeResponse.thumbprint_hash_function) diff --git a/acme/acme/client.py b/acme/acme/client.py index f5aa1ff9e..548c3d548 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -8,10 +8,12 @@ import http.client as http_client import logging import re import time +from typing import cast from typing import Dict from typing import List from typing import Set from typing import Text +from typing import Union import josepy as jose import OpenSSL @@ -251,7 +253,7 @@ class Client(ClientBase): if isinstance(directory, str): directory = messages.Directory.from_json( net.get(directory).json()) - super(Client, self).__init__(directory=directory, + super().__init__(directory=directory, net=net, acme_version=1) def register(self, new_reg=None): @@ -575,7 +577,7 @@ class ClientV2(ClientBase): :param .messages.Directory directory: Directory Resource :param .ClientNetwork net: Client network. """ - super(ClientV2, self).__init__(directory=directory, + super().__init__(directory=directory, net=net, acme_version=2) def new_account(self, new_account): @@ -625,7 +627,7 @@ class ClientV2(ClientBase): """ # https://github.com/certbot/certbot/issues/6155 new_regr = self._get_v2_account(regr) - return super(ClientV2, self).update_registration(new_regr, update) + return super().update_registration(new_regr, update) def _get_v2_account(self, regr): self.net.account = None @@ -818,6 +820,7 @@ class BackwardsCompatibleClientV2: def __init__(self, net, key, server): directory = messages.Directory.from_json(net.get(server).json()) self.acme_version = self._acme_version_from_directory(directory) + self.client: Union[Client, ClientV2] if self.acme_version == 1: self.client = Client(directory, key=key, net=net) else: @@ -837,16 +840,18 @@ class BackwardsCompatibleClientV2: if check_tos_cb is not None: check_tos_cb(tos) if self.acme_version == 1: - regr = self.client.register(regr) + client_v1 = cast(Client, self.client) + regr = client_v1.register(regr) if regr.terms_of_service is not None: _assess_tos(regr.terms_of_service) - return self.client.agree_to_tos(regr) + return client_v1.agree_to_tos(regr) return regr else: - if "terms_of_service" in self.client.directory.meta: - _assess_tos(self.client.directory.meta.terms_of_service) + client_v2 = cast(ClientV2, self.client) + if "terms_of_service" in client_v2.directory.meta: + _assess_tos(client_v2.directory.meta.terms_of_service) regr = regr.update(terms_of_service_agreed=True) - return self.client.new_account(regr) + return client_v2.new_account(regr) def new_order(self, csr_pem): """Request a new Order object from the server. @@ -864,14 +869,15 @@ class BackwardsCompatibleClientV2: """ if self.acme_version == 1: + client_v1 = cast(Client, self.client) csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) # pylint: disable=protected-access dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) authorizations = [] for domain in dnsNames: - authorizations.append(self.client.request_domain_challenges(domain)) + authorizations.append(client_v1.request_domain_challenges(domain)) return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem) - return self.client.new_order(csr_pem) + return cast(ClientV2, self.client).new_order(csr_pem) def finalize_order(self, orderr, deadline, fetch_alternative_chains=False): """Finalize an order and obtain a certificate. @@ -886,8 +892,9 @@ class BackwardsCompatibleClientV2: """ if self.acme_version == 1: + client_v1 = cast(Client, self.client) csr_pem = orderr.csr_pem - certr = self.client.request_issuance( + certr = client_v1.request_issuance( jose.ComparableX509( OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)), orderr.authorizations) @@ -895,7 +902,7 @@ class BackwardsCompatibleClientV2: chain = None while datetime.datetime.now() < deadline: try: - chain = self.client.fetch_chain(certr) + chain = client_v1.fetch_chain(certr) break except errors.Error: time.sleep(1) @@ -910,7 +917,8 @@ class BackwardsCompatibleClientV2: chain = crypto_util.dump_pyopenssl_chain(chain).decode() return orderr.update(fullchain_pem=(cert + chain)) - return self.client.finalize_order(orderr, deadline, fetch_alternative_chains) + return cast(ClientV2, self.client).finalize_order( + orderr, deadline, fetch_alternative_chains) def revoke(self, cert, rsn): """Revoke certificate. @@ -936,7 +944,7 @@ class BackwardsCompatibleClientV2: Always return False for ACMEv1 servers, as it doesn't use External Account Binding.""" if self.acme_version == 1: return False - return self.client.external_account_required() + return cast(ClientV2, self.client).external_account_required() class ClientNetwork: @@ -1129,6 +1137,7 @@ class ClientNetwork: # If content is DER, log the base64 of it instead of raw bytes, to keep # binary data out of the logs. + debug_content: Union[bytes, str] if response.headers.get("Content-Type") == DER_CONTENT_TYPE: debug_content = base64.b64encode(response.content) else: diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 5ca5a4fa2..b47ed88da 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -28,13 +28,8 @@ class NonceError(ClientError): class BadNonce(NonceError): """Bad nonce error.""" - def __init__(self, nonce, error, *args, **kwargs): - # MyPy complains here that there is too many arguments for BaseException constructor. - # This is an error fixed in typeshed, see https://github.com/python/mypy/issues/4183 - # The fix is included in MyPy>=0.740, but upgrading it would bring dozen of errors due to - # new types definitions. So we ignore the error until the code base is fixed to match - # with MyPy>=0.740 referential. - super(BadNonce, self).__init__(*args, **kwargs) # type: ignore + def __init__(self, nonce, error, *args): + super().__init__(*args) self.nonce = nonce self.error = error @@ -52,9 +47,8 @@ class MissingNonce(NonceError): :ivar requests.Response ~.response: HTTP Response """ - def __init__(self, response, *args, **kwargs): - # See comment in BadNonce constructor above for an explanation of type: ignore here. - super(MissingNonce, self).__init__(*args, **kwargs) # type: ignore + def __init__(self, response, *args): + super().__init__(*args) self.response = response def __str__(self): @@ -78,7 +72,7 @@ class PollError(ClientError): def __init__(self, exhausted, updated): self.exhausted = exhausted self.updated = updated - super(PollError, self).__init__() + super().__init__() @property def timeout(self): @@ -96,7 +90,7 @@ class ValidationError(Error): """ def __init__(self, failed_authzrs): self.failed_authzrs = failed_authzrs - super(ValidationError, self).__init__() + super().__init__() class TimeoutError(Error): # pylint: disable=redefined-builtin @@ -112,7 +106,7 @@ class IssuanceError(Error): :param messages.Error error: The error provided by the server. """ self.error = error - super(IssuanceError, self).__init__() + super().__init__() class ConflictError(ClientError): @@ -125,7 +119,7 @@ class ConflictError(ClientError): """ def __init__(self, location): self.location = location - super(ConflictError, self).__init__() + super().__init__() class WildcardUnsupportedError(Error): diff --git a/acme/acme/fields.py b/acme/acme/fields.py index 3b5672283..bd915e47d 100644 --- a/acme/acme/fields.py +++ b/acme/acme/fields.py @@ -12,7 +12,7 @@ class Fixed(jose.Field): def __init__(self, json_name, value): self.value = value - super(Fixed, self).__init__( + super().__init__( json_name=json_name, default=value, omitempty=False) def decode(self, value): @@ -53,7 +53,7 @@ class Resource(jose.Field): def __init__(self, resource_type, *args, **kwargs): self.resource_type = resource_type - super(Resource, self).__init__( + super().__init__( 'resource', default=resource_type, *args, **kwargs) def decode(self, value): diff --git a/acme/acme/jws.py b/acme/acme/jws.py index 2188c3727..d9d3e12c3 100644 --- a/acme/acme/jws.py +++ b/acme/acme/jws.py @@ -14,7 +14,9 @@ class Header(jose.Header): kid = jose.Field('kid', omitempty=True) url = jose.Field('url', omitempty=True) - @nonce.decoder + # Mypy does not understand the josepy magic happening here, and falsely claims + # that nonce is redefined. Let's ignore the type check here. + @nonce.decoder # type: ignore def nonce(value): # pylint: disable=no-self-argument,missing-function-docstring try: return jose.decode_b64jose(value) @@ -48,7 +50,7 @@ class JWS(jose.JWS): # Per ACME spec, jwk and kid are mutually exclusive, so only include a # jwk field if kid is not provided. include_jwk = kid is None - return super(JWS, cls).sign(payload, key=key, alg=alg, + return super().sign(payload, key=key, alg=alg, protect=frozenset(['nonce', 'url', 'kid', 'jwk', 'alg']), nonce=nonce, url=url, kid=kid, include_jwk=include_jwk) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 5e7e22c34..36207dba0 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,6 +1,9 @@ """ACME protocol messages.""" -import json from collections.abc import Hashable +import json +from typing import Any +from typing import Dict +from typing import Type import josepy as jose @@ -87,7 +90,9 @@ class Error(jose.JSONObjectWithFields, errors.Error): raise ValueError("The supplied code: %s is not a known ACME error" " code" % code) typ = ERROR_PREFIX + code - return cls(typ=typ, **kwargs) + # Mypy will not understand that the Error constructor accepts a named argument + # "typ" because of josepy magic. Let's ignore the type check here. + return cls(typ=typ, **kwargs) # type: ignore @property def description(self): @@ -124,10 +129,10 @@ class Error(jose.JSONObjectWithFields, errors.Error): class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore """ACME constant.""" __slots__ = ('name',) - POSSIBLE_NAMES = NotImplemented + POSSIBLE_NAMES: Dict[str, '_Constant'] = NotImplemented def __init__(self, name): - super(_Constant, self).__init__() + super().__init__() self.POSSIBLE_NAMES[name] = self # pylint: disable=unsupported-assignment-operation self.name = name @@ -166,7 +171,7 @@ STATUS_DEACTIVATED = Status('deactivated') class IdentifierType(_Constant): """ACME identifier type.""" - POSSIBLE_NAMES: dict = {} + POSSIBLE_NAMES: Dict[str, 'IdentifierType'] = {} IDENTIFIER_FQDN = IdentifierType('dns') # IdentifierDNS in Boulder @@ -184,7 +189,7 @@ class Identifier(jose.JSONObjectWithFields): class Directory(jose.JSONDeSerializable): """Directory.""" - _REGISTERED_TYPES: dict = {} + _REGISTERED_TYPES: Dict[str, Type[Any]] = {} class Meta(jose.JSONObjectWithFields): """Directory Meta.""" @@ -196,7 +201,7 @@ class Directory(jose.JSONDeSerializable): def __init__(self, **kwargs): kwargs = {self._internal_name(k): v for k, v in kwargs.items()} - super(Directory.Meta, self).__init__(**kwargs) + super().__init__(**kwargs) @property def terms_of_service(self): @@ -206,7 +211,7 @@ class Directory(jose.JSONDeSerializable): def __iter__(self): # When iterating over fields, use the external name 'terms_of_service' instead of # the internal '_terms_of_service'. - for name in super(Directory.Meta, self).__iter__(): + for name in super().__iter__(): yield name[1:] if name == '_terms_of_service' else name def _internal_name(self, name): @@ -218,7 +223,7 @@ class Directory(jose.JSONDeSerializable): return getattr(key, 'resource_type', key) @classmethod - def register(cls, resource_body_cls): + def register(cls, resource_body_cls: Type[Any]) -> Type[Any]: """Register resource.""" resource_type = resource_body_cls.resource_type assert resource_type not in cls._REGISTERED_TYPES @@ -352,7 +357,7 @@ class Registration(ResourceBody): if 'contact' in kwargs: # Avoid the __setattr__ used by jose.TypedJSONObjectWithFields object.__setattr__(self, '_add_contact', True) - super(Registration, self).__init__(**kwargs) + super().__init__(**kwargs) def _filter_contact(self, prefix): return tuple( @@ -378,12 +383,12 @@ class Registration(ResourceBody): def to_partial_json(self): """Modify josepy.JSONDeserializable.to_partial_json()""" - jobj = super(Registration, self).to_partial_json() + jobj = super().to_partial_json() return self._add_contact_if_appropriate(jobj) def fields_to_partial_json(self): """Modify josepy.JSONObjectWithFields.fields_to_partial_json()""" - jobj = super(Registration, self).fields_to_partial_json() + jobj = super().fields_to_partial_json() return self._add_contact_if_appropriate(jobj) @property @@ -455,19 +460,19 @@ class ChallengeBody(ResourceBody): def __init__(self, **kwargs): kwargs = {self._internal_name(k): v for k, v in kwargs.items()} - super(ChallengeBody, self).__init__(**kwargs) + super().__init__(**kwargs) def encode(self, name): - return super(ChallengeBody, self).encode(self._internal_name(name)) + return super().encode(self._internal_name(name)) def to_partial_json(self): - jobj = super(ChallengeBody, self).to_partial_json() + jobj = super().to_partial_json() jobj.update(self.chall.to_partial_json()) return jobj @classmethod def fields_from_json(cls, jobj): - jobj_fields = super(ChallengeBody, cls).fields_from_json(jobj) + jobj_fields = super().fields_from_json(jobj) jobj_fields['chall'] = challenges.Challenge.from_json(jobj) return jobj_fields @@ -482,7 +487,7 @@ class ChallengeBody(ResourceBody): def __iter__(self): # When iterating over fields, use the external name 'uri' instead of # the internal '_uri'. - for name in super(ChallengeBody, self).__iter__(): + for name in super().__iter__(): yield name[1:] if name == '_uri' else name def _internal_name(self, name): @@ -528,7 +533,9 @@ class Authorization(ResourceBody): expires = fields.RFC3339Field('expires', omitempty=True) wildcard = jose.Field('wildcard', omitempty=True) - @challenges.decoder + # Mypy does not understand the josepy magic happening here, and falsely claims + # that challenge is redefined. Let's ignore the type check here. + @challenges.decoder # type: ignore def challenges(value): # pylint: disable=no-self-argument,missing-function-docstring return tuple(ChallengeBody.from_json(chall) for chall in value) @@ -627,7 +634,9 @@ class Order(ResourceBody): expires = fields.RFC3339Field('expires', omitempty=True) error = jose.Field('error', omitempty=True, decoder=Error.from_json) - @identifiers.decoder + # Mypy does not understand the josepy magic happening here, and falsely claims + # that identifiers is redefined. Let's ignore the type check here. + @identifiers.decoder # type: ignore def identifiers(value): # pylint: disable=no-self-argument,missing-function-docstring return tuple(Identifier.from_json(identifier) for identifier in value) diff --git a/acme/acme/mixins.py b/acme/acme/mixins.py index 0e1e0977c..6d58e0889 100644 --- a/acme/acme/mixins.py +++ b/acme/acme/mixins.py @@ -20,7 +20,7 @@ class VersionedLEACMEMixin: # Required for @property to operate properly. See comment above. object.__setattr__(self, key, value) else: - super(VersionedLEACMEMixin, self).__setattr__(key, value) # pragma: no cover + super().__setattr__(key, value) # pragma: no cover class ResourceMixin(VersionedLEACMEMixin): @@ -30,12 +30,12 @@ class ResourceMixin(VersionedLEACMEMixin): """ def to_partial_json(self): """See josepy.JSONDeserializable.to_partial_json()""" - return _safe_jobj_compliance(super(ResourceMixin, self), + return _safe_jobj_compliance(super(), 'to_partial_json', 'resource') def fields_to_partial_json(self): """See josepy.JSONObjectWithFields.fields_to_partial_json()""" - return _safe_jobj_compliance(super(ResourceMixin, self), + return _safe_jobj_compliance(super(), 'fields_to_partial_json', 'resource') @@ -46,12 +46,12 @@ class TypeMixin(VersionedLEACMEMixin): """ def to_partial_json(self): """See josepy.JSONDeserializable.to_partial_json()""" - return _safe_jobj_compliance(super(TypeMixin, self), + return _safe_jobj_compliance(super(), 'to_partial_json', 'type') def fields_to_partial_json(self): """See josepy.JSONObjectWithFields.fields_to_partial_json()""" - return _safe_jobj_compliance(super(TypeMixin, self), + return _safe_jobj_compliance(super(), 'fields_to_partial_json', 'type') diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 0b5a8b5c7..eda45304c 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -64,7 +64,7 @@ class BaseDualNetworkedServers: def __init__(self, ServerClass, server_address, *remaining_args, **kwargs): port = server_address[1] self.threads: List[threading.Thread] = [] - self.servers: List[ACMEServerMixin] = [] + self.servers: List[socketserver.BaseServer] = [] # Must try True first. # Ubuntu, for example, will fail to bind to IPv4 if we've already bound @@ -203,8 +203,24 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.simple_http_resources = kwargs.pop("simple_http_resources", set()) - self.timeout = kwargs.pop('timeout', 30) + self._timeout = kwargs.pop('timeout', 30) BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + self.server: HTTP01Server + + # In parent class BaseHTTPRequestHandler, 'timeout' is a class-level property but we + # need to define its value during the initialization phase in HTTP01RequestHandler. + # However MyPy does not appreciate that we dynamically shadow a class-level property + # with an instance-level property (eg. self.timeout = ... in __init__()). So to make + # everyone happy, we statically redefine 'timeout' as a method property, and set the + # timeout value in a new internal instance-level property _timeout. + @property + def timeout(self): + """ + The default timeout this server should apply to requests. + :return: timeout to apply + :rtype: int + """ + return self._timeout def log_message(self, format, *args): # pylint: disable=redefined-builtin """Log arbitrary message.""" diff --git a/acme/setup.py b/acme/setup.py index 90ab5c9f3..6fa49dafc 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -37,7 +37,7 @@ setup( description='ACME protocol implementation in Python', url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index cc604b0de..71bf61976 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -292,7 +292,7 @@ class TLSALPN01ResponseTest(unittest.TestCase): def test_gen_verify_cert_gen_key(self): cert, key = self.response.gen_cert(self.domain) - self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) + self.assertIsInstance(key, OpenSSL.crypto.PKey) self.assertTrue(self.response.verify_cert(self.domain, cert)) def test_verify_bad_cert(self): @@ -431,7 +431,7 @@ class DNSTest(unittest.TestCase): mock_gen.return_value = mock.sentinel.validation response = self.msg.gen_response(KEY) from acme.challenges import DNSResponse - self.assertTrue(isinstance(response, DNSResponse)) + self.assertIsInstance(response, DNSResponse) self.assertEqual(response.validation, mock.sentinel.validation) def test_validation_domain_name(self): diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 14247335c..35cc0ba25 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -5,6 +5,7 @@ import datetime import http.client as http_client import json import unittest +from typing import Dict from unittest import mock import josepy as jose @@ -89,7 +90,7 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" def setUp(self): - super(BackwardsCompatibleClientV2Test, self).setUp() + super().setUp() # contains a loaded cert self.certr = messages.CertificateResource( body=messages_test.CERT) @@ -318,7 +319,7 @@ class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" def setUp(self): - super(ClientTest, self).setUp() + super().setUp() self.directory = DIRECTORY_V1 @@ -603,8 +604,8 @@ class ClientTest(ClientTestBase): # make sure that max_attempts is per-authorization, rather # than global max_attempts=max(len(authzrs[0].retries), len(authzrs[1].retries))) - self.assertTrue(cert[0] is csr) - self.assertTrue(cert[1] is updated_authzrs) + self.assertIs(cert[0], csr) + self.assertIs(cert[1], updated_authzrs) self.assertEqual(updated_authzrs[0].uri, 'a...') self.assertEqual(updated_authzrs[1].uri, 'b.') self.assertEqual(updated_authzrs[0].times, [ @@ -640,7 +641,7 @@ class ClientTest(ClientTestBase): authzr = self.client.deactivate_authorization(self.authzr) self.assertEqual(authzb, authzr.body) self.assertEqual(self.client.net.post.call_count, 1) - self.assertTrue(self.authzr.uri in self.net.post.call_args_list[0][0]) + self.assertIn(self.authzr.uri, self.net.post.call_args_list[0][0]) def test_check_cert(self): self.response.headers['Location'] = self.certr.uri @@ -699,7 +700,7 @@ class ClientTest(ClientTestBase): def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) - self.assertTrue('reason' in obj.to_partial_json().keys()) + self.assertIn('reason', obj.to_partial_json().keys()) self.assertEqual(self.rsn, obj.to_partial_json()['reason']) def test_revoke_bad_status_raises_error(self): @@ -715,7 +716,7 @@ class ClientV2Test(ClientTestBase): """Tests for acme.client.ClientV2.""" def setUp(self): - super(ClientV2Test, self).setUp() + super().setUp() self.directory = DIRECTORY_V2 @@ -876,9 +877,9 @@ class ClientV2Test(ClientTestBase): self.response.headers['Location'] = self.regr.uri self.response.json.return_value = self.regr.body.to_json() self.assertEqual(self.regr, self.client.update_registration(self.regr)) - self.assertNotEqual(self.client.net.account, None) + self.assertIsNotNone(self.client.net.account) self.assertEqual(self.client.net.post.call_count, 2) - self.assertTrue(DIRECTORY_V2.newAccount in self.net.post.call_args_list[0][0]) + self.assertIn(DIRECTORY_V2.newAccount, self.net.post.call_args_list[0][0]) self.response.json.return_value = self.regr.body.update( contact=()).to_json() @@ -942,7 +943,7 @@ class ClientNetworkTest(unittest.TestCase): self.response.links = {} def test_init(self): - self.assertTrue(self.net.verify_ssl is self.verify_ssl) + self.assertIs(self.net.verify_ssl, self.verify_ssl) def test_wrap_in_jws(self): # pylint: disable=protected-access @@ -1184,7 +1185,7 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): def send_request(*args, **kwargs): # pylint: disable=unused-argument,missing-docstring - self.assertFalse("new_nonce_url" in kwargs) + self.assertNotIn("new_nonce_url", kwargs) method = args[0] uri = args[1] if method == 'HEAD' and uri != "new_nonce_uri": @@ -1329,7 +1330,7 @@ class ClientNetworkSourceAddressBindingTest(unittest.TestCase): from acme.client import ClientNetwork net = ClientNetwork(key=None, alg=None, source_address=self.source_address) for adapter in net.session.adapters.values(): - self.assertTrue(self.source_address in adapter.source_address) + self.assertIn(self.source_address, adapter.source_address) def test_behavior_assumption(self): """This is a test that guardrails the HTTPAdapter behavior so that if the default for diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index 8075b68ed..cc81a2f1f 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -5,6 +5,7 @@ import socketserver import threading import time import unittest +from typing import List import josepy as jose import OpenSSL @@ -190,7 +191,7 @@ class RandomSnTest(unittest.TestCase): for _ in range(self.cert_count): cert = gen_ss_cert(self.key, ['dummy'], force_san=True) self.serial_num.append(cert.get_serial_number()) - self.assertTrue(len(set(self.serial_num)) > 1) + self.assertGreater(len(set(self.serial_num)), 1) class MakeCSRTest(unittest.TestCase): """Test for standalone functions.""" @@ -205,8 +206,8 @@ class MakeCSRTest(unittest.TestCase): def test_make_csr(self): csr_pem = self._call_with_key(["a.example", "b.example"]) - self.assertTrue(b'--BEGIN CERTIFICATE REQUEST--' in csr_pem) - self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) + self.assertIn(b'--BEGIN CERTIFICATE REQUEST--', csr_pem) + self.assertIn(b'--END CERTIFICATE REQUEST--', csr_pem) csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index 11c57059c..f325b284e 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -24,8 +24,8 @@ class MissingNonceTest(unittest.TestCase): self.error = MissingNonce(self.response) def test_str(self): - self.assertTrue("FOO" in str(self.error)) - self.assertTrue("{}" in str(self.error)) + self.assertIn("FOO", str(self.error)) + self.assertIn("{}", str(self.error)) class PollErrorTest(unittest.TestCase): diff --git a/acme/tests/jws_test.py b/acme/tests/jws_test.py index 2e6ad72dd..0787fb340 100644 --- a/acme/tests/jws_test.py +++ b/acme/tests/jws_test.py @@ -48,7 +48,7 @@ class JWSTest(unittest.TestCase): self.assertEqual(jws.signature.combined.nonce, self.nonce) self.assertEqual(jws.signature.combined.url, self.url) self.assertEqual(jws.signature.combined.kid, self.kid) - self.assertEqual(jws.signature.combined.jwk, None) + self.assertIsNone(jws.signature.combined.jwk) # TODO: check that nonce is in protected header self.assertEqual(jws, JWS.from_json(jws.to_json())) @@ -58,7 +58,7 @@ class JWSTest(unittest.TestCase): jws = JWS.sign(payload=b'foo', key=self.privkey, alg=jose.RS256, nonce=self.nonce, url=self.url) - self.assertEqual(jws.signature.combined.kid, None) + self.assertIsNone(jws.signature.combined.kid) self.assertEqual(jws.signature.combined.jwk, self.pubkey) diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index 99a3a9ce4..3f0f29215 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -41,13 +41,13 @@ class ErrorTest(unittest.TestCase): def test_description(self): self.assertEqual('The request message was malformed', self.error.description) - self.assertTrue(self.error_custom.description is None) + self.assertIsNone(self.error_custom.description) def test_code(self): from acme.messages import Error self.assertEqual('malformed', self.error.code) - self.assertEqual(None, self.error_custom.code) - self.assertEqual(None, Error().code) + self.assertIsNone(self.error_custom.code) + self.assertIsNone(Error().code) def test_is_acme_error(self): from acme.messages import is_acme_error, Error @@ -260,10 +260,10 @@ class RegistrationTest(unittest.TestCase): self.assertEqual(empty_new_reg.contact, ()) self.assertEqual(new_reg_with_contact.contact, ()) - self.assertTrue('contact' not in empty_new_reg.to_partial_json()) - self.assertTrue('contact' not in empty_new_reg.fields_to_partial_json()) - self.assertTrue('contact' in new_reg_with_contact.to_partial_json()) - self.assertTrue('contact' in new_reg_with_contact.fields_to_partial_json()) + self.assertNotIn('contact', empty_new_reg.to_partial_json()) + self.assertNotIn('contact', empty_new_reg.fields_to_partial_json()) + self.assertIn('contact', new_reg_with_contact.to_partial_json()) + self.assertIn('contact', new_reg_with_contact.fields_to_partial_json()) class UpdateRegistrationTest(unittest.TestCase): @@ -406,7 +406,7 @@ class AuthorizationResourceTest(unittest.TestCase): authzr = AuthorizationResource( uri=mock.sentinel.uri, body=mock.sentinel.body) - self.assertTrue(isinstance(authzr, jose.JSONDeSerializable)) + self.assertIsInstance(authzr, jose.JSONDeSerializable) class CertificateRequestTest(unittest.TestCase): @@ -417,7 +417,7 @@ class CertificateRequestTest(unittest.TestCase): self.req = CertificateRequest(csr=CSR) def test_json_de_serializable(self): - self.assertTrue(isinstance(self.req, jose.JSONDeSerializable)) + self.assertIsInstance(self.req, jose.JSONDeSerializable) from acme.messages import CertificateRequest self.assertEqual( self.req, CertificateRequest.from_json(self.req.to_json())) @@ -433,7 +433,7 @@ class CertificateResourceTest(unittest.TestCase): cert_chain_uri=mock.sentinel.cert_chain_uri) def test_json_de_serializable(self): - self.assertTrue(isinstance(self.certr, jose.JSONDeSerializable)) + self.assertIsInstance(self.certr, jose.JSONDeSerializable) from acme.messages import CertificateResource self.assertEqual( self.certr, CertificateResource.from_json(self.certr.to_json())) diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index a02d008a0..17d73dba8 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -4,6 +4,7 @@ import socket import socketserver import threading import unittest +from typing import Set from unittest import mock import josepy as jose diff --git a/certbot-apache/certbot_apache/_internal/apacheparser.py b/certbot-apache/certbot_apache/_internal/apacheparser.py index c7b723ae6..d3bd1a4bf 100644 --- a/certbot-apache/certbot_apache/_internal/apacheparser.py +++ b/certbot-apache/certbot_apache/_internal/apacheparser.py @@ -1,4 +1,5 @@ """ apacheconfig implementation of the ParserNode interfaces """ +from typing import Tuple from certbot_apache._internal import assertions from certbot_apache._internal import interfaces @@ -14,14 +15,14 @@ class ApacheParserNode(interfaces.ParserNode): def __init__(self, **kwargs): ancestor, dirty, filepath, metadata = util.parsernode_kwargs(kwargs) # pylint: disable=unused-variable - super(ApacheParserNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.ancestor = ancestor self.filepath = filepath self.dirty = dirty self.metadata = metadata self._raw = self.metadata["ac_ast"] - def save(self, msg): # pragma: no cover + def save(self, msg): # pragma: no cover pass def find_ancestors(self, name): # pylint: disable=unused-variable @@ -38,7 +39,7 @@ class ApacheCommentNode(ApacheParserNode): def __init__(self, **kwargs): comment, kwargs = util.commentnode_kwargs(kwargs) # pylint: disable=unused-variable - super(ApacheCommentNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.comment = comment def __eq__(self, other): # pragma: no cover @@ -56,7 +57,7 @@ class ApacheDirectiveNode(ApacheParserNode): def __init__(self, **kwargs): name, parameters, enabled, kwargs = util.directivenode_kwargs(kwargs) - super(ApacheDirectiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.name = name self.parameters = parameters self.enabled = enabled @@ -82,8 +83,8 @@ class ApacheBlockNode(ApacheDirectiveNode): """ apacheconfig implementation of BlockNode interface """ def __init__(self, **kwargs): - super(ApacheBlockNode, self).__init__(**kwargs) - self.children = () + super().__init__(**kwargs) + self.children: Tuple[ApacheParserNode, ...] = () def __eq__(self, other): # pragma: no cover if isinstance(other, self.__class__): diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index 4549e935a..896e17cf8 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -80,7 +80,7 @@ class AugeasParserNode(interfaces.ParserNode): def __init__(self, **kwargs): ancestor, dirty, filepath, metadata = util.parsernode_kwargs(kwargs) # pylint: disable=unused-variable - super(AugeasParserNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.ancestor = ancestor self.filepath = filepath self.dirty = dirty @@ -169,7 +169,7 @@ class AugeasCommentNode(AugeasParserNode): def __init__(self, **kwargs): comment, kwargs = util.commentnode_kwargs(kwargs) # pylint: disable=unused-variable - super(AugeasCommentNode, self).__init__(**kwargs) + super().__init__(**kwargs) # self.comment = comment self.comment = comment @@ -188,7 +188,7 @@ class AugeasDirectiveNode(AugeasParserNode): def __init__(self, **kwargs): name, parameters, enabled, kwargs = util.directivenode_kwargs(kwargs) - super(AugeasDirectiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.name = name self.enabled = enabled if parameters: @@ -245,7 +245,7 @@ class AugeasBlockNode(AugeasDirectiveNode): """ Augeas implementation of BlockNode interface """ def __init__(self, **kwargs): - super(AugeasBlockNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.children = () def __eq__(self, other): diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 29648d4c1..abf199793 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -11,6 +11,7 @@ import time from typing import DefaultDict from typing import Dict from typing import List +from typing import Optional from typing import Set from typing import Union @@ -35,6 +36,9 @@ from certbot_apache._internal import dualparser from certbot_apache._internal import http_01 from certbot_apache._internal import obj from certbot_apache._internal import parser +from certbot_apache._internal.dualparser import DualBlockNode +from certbot_apache._internal.obj import VirtualHost +from certbot_apache._internal.parser import ApacheParser try: import apacheconfig @@ -46,6 +50,47 @@ except ImportError: # pragma: no cover logger = logging.getLogger(__name__) +class OsOptions: + """ + Dedicated class to describe the OS specificities (eg. paths, binary names) + that the Apache configurator needs to be aware to operate properly. + """ + def __init__(self, + server_root="/etc/apache2", + vhost_root="/etc/apache2/sites-available", + vhost_files="*", + logs_root="/var/log/apache2", + ctl="apache2ctl", + version_cmd: Optional[List[str]] = None, + restart_cmd: Optional[List[str]] = None, + restart_cmd_alt: Optional[List[str]] = None, + conftest_cmd: Optional[List[str]] = None, + enmod: Optional[str] = None, + dismod: Optional[str] = None, + le_vhost_ext="-le-ssl.conf", + handle_modules=False, + handle_sites=False, + challenge_location="/etc/apache2", + apache_bin: Optional[str] = None, + ): + self.server_root = server_root + self.vhost_root = vhost_root + self.vhost_files = vhost_files + self.logs_root = logs_root + self.ctl = ctl + self.version_cmd = ['apache2ctl', '-v'] if not version_cmd else version_cmd + self.restart_cmd = ['apache2ctl', 'graceful'] if not restart_cmd else restart_cmd + self.restart_cmd_alt = restart_cmd_alt + self.conftest_cmd = ['apache2ctl', 'configtest'] if not conftest_cmd else conftest_cmd + self.enmod = enmod + self.dismod = dismod + self.le_vhost_ext = le_vhost_ext + self.handle_modules = handle_modules + self.handle_sites = handle_sites + self.challenge_location = challenge_location + self.bin = apache_bin + + # TODO: Augeas sections ie. , beginning and closing # tags need to be the same case, otherwise Augeas doesn't recognize them. # This is not able to be completely remedied by regular expressions because @@ -101,27 +146,7 @@ class ApacheConfigurator(common.Installer): " change depending on the operating system Certbot is run on.)" ) - OS_DEFAULTS = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/sites-available", - vhost_files="*", - logs_root="/var/log/apache2", - ctl="apache2ctl", - version_cmd=['apache2ctl', '-v'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, - challenge_location="/etc/apache2", - bin=None - ) - - def option(self, key): - """Get a value from options""" - return self.options.get(key) + OS_DEFAULTS = OsOptions() def pick_apache_config(self, warn_on_no_mod_ssl=True): """ @@ -151,14 +176,14 @@ class ApacheConfigurator(common.Installer): for o in opts: # Config options use dashes instead of underscores if self.conf(o.replace("_", "-")) is not None: - self.options[o] = self.conf(o.replace("_", "-")) + setattr(self.options, o, self.conf(o.replace("_", "-"))) else: - self.options[o] = self.OS_DEFAULTS[o] + setattr(self.options, o, getattr(self.OS_DEFAULTS, o)) # Special cases - self.options["version_cmd"][0] = self.option("ctl") - self.options["restart_cmd"][0] = self.option("ctl") - self.options["conftest_cmd"][0] = self.option("ctl") + self.options.version_cmd[0] = self.options.ctl + self.options.restart_cmd[0] = self.options.ctl + self.options.conftest_cmd[0] = self.options.ctl @classmethod def add_parser_arguments(cls, add): @@ -173,30 +198,30 @@ class ApacheConfigurator(common.Installer): else: # cls.OS_DEFAULTS can be distribution specific, see override classes DEFAULTS = cls.OS_DEFAULTS - add("enmod", default=DEFAULTS["enmod"], + add("enmod", default=DEFAULTS.enmod, help="Path to the Apache 'a2enmod' binary") - add("dismod", default=DEFAULTS["dismod"], + add("dismod", default=DEFAULTS.dismod, help="Path to the Apache 'a2dismod' binary") - add("le-vhost-ext", default=DEFAULTS["le_vhost_ext"], + add("le-vhost-ext", default=DEFAULTS.le_vhost_ext, help="SSL vhost configuration extension") - add("server-root", default=DEFAULTS["server_root"], + add("server-root", default=DEFAULTS.server_root, help="Apache server root directory") add("vhost-root", default=None, help="Apache server VirtualHost configuration root") - add("logs-root", default=DEFAULTS["logs_root"], + add("logs-root", default=DEFAULTS.logs_root, help="Apache server logs directory") add("challenge-location", - default=DEFAULTS["challenge_location"], + default=DEFAULTS.challenge_location, help="Directory path for challenge configuration") - add("handle-modules", default=DEFAULTS["handle_modules"], + add("handle-modules", default=DEFAULTS.handle_modules, help="Let installer handle enabling required modules for you " + "(Only Ubuntu/Debian currently)") - add("handle-sites", default=DEFAULTS["handle_sites"], + add("handle-sites", default=DEFAULTS.handle_sites, help="Let installer handle enabling sites for you " + "(Only Ubuntu/Debian currently)") - add("ctl", default=DEFAULTS["ctl"], + add("ctl", default=DEFAULTS.ctl, help="Full path to Apache control script") - add("bin", default=DEFAULTS["bin"], + add("bin", default=DEFAULTS.bin, help="Full path to apache2/httpd binary") def __init__(self, *args, **kwargs): @@ -209,7 +234,7 @@ class ApacheConfigurator(common.Installer): version = kwargs.pop("version", None) use_parsernode = kwargs.pop("use_parsernode", False) openssl_version = kwargs.pop("openssl_version", None) - super(ApacheConfigurator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) # Add name_server association dict self.assoc: Dict[str, obj.VirtualHost] = {} @@ -231,11 +256,11 @@ class ApacheConfigurator(common.Installer): self.parsed_paths: List[str] = [] # These will be set in the prepare function self._prepared = False - self.parser = None - self.parser_root = None + self.parser: ApacheParser + self.parser_root: Optional[DualBlockNode] = None self.version = version self._openssl_version = openssl_version - self.vhosts = None + self.vhosts: List[VirtualHost] self.options = copy.deepcopy(self.OS_DEFAULTS) self._enhance_func = {"redirect": self._enable_redirect, "ensure-http-header": self._set_http_header, @@ -285,8 +310,8 @@ class ApacheConfigurator(common.Installer): ssl_module_location = self.parser.standard_path_from_server_root(ssl_module_location) else: # Possibility B: ssl_module is statically linked into Apache - if self.option("bin"): - ssl_module_location = self.option("bin") + if self.options.bin: + ssl_module_location = self.options.bin else: logger.warning("ssl_module is statically linked but --apache-bin is " "missing; not disabling session tickets.") @@ -316,7 +341,7 @@ class ApacheConfigurator(common.Installer): self._prepare_options() # Verify Apache is installed - self._verify_exe_availability(self.option("ctl")) + self._verify_exe_availability(self.options.ctl) # Make sure configuration is valid self.config_test() @@ -344,8 +369,9 @@ class ApacheConfigurator(common.Installer): "augeaspath": self.parser.get_root_augpath(), "ac_ast": None} if self.USE_PARSERNODE: - self.parser_root = self.get_parsernode_root(pn_meta) - self.parsed_paths = self.parser_root.parsed_paths() + parser_root = self.get_parsernode_root(pn_meta) + self.parser_root = parser_root + self.parsed_paths = parser_root.parsed_paths() # Check for errors in parsing files with Augeas self.parser.check_parsing_errors("httpd.aug") @@ -355,20 +381,20 @@ class ApacheConfigurator(common.Installer): # We may try to enable mod_ssl later. If so, we shouldn't warn if we can't find it now. # This is currently only true for debian/ubuntu. - warn_on_no_mod_ssl = not self.option("handle_modules") + warn_on_no_mod_ssl = not self.options.handle_modules self.install_ssl_options_conf(self.mod_ssl_conf, self.updated_mod_ssl_conf_digest, warn_on_no_mod_ssl) # Prevent two Apache plugins from modifying a config at once try: - util.lock_dir_until_exit(self.option("server_root")) + util.lock_dir_until_exit(self.options.server_root) except (OSError, errors.LockError): logger.debug("Encountered error:", exc_info=True) raise errors.PluginError( "Unable to create a lock file in {0}. Are you running" " Certbot with sufficient privileges to modify your" - " Apache configuration?".format(self.option("server_root"))) + " Apache configuration?".format(self.options.server_root)) self._prepared = True def save(self, title=None, temporary=False): @@ -404,10 +430,10 @@ class ApacheConfigurator(common.Installer): :raises .errors.PluginError: If unable to recover the configuration """ - super(ApacheConfigurator, self).recovery_routine() + super().recovery_routine() # Reload configuration after these changes take effect if needed # ie. ApacheParser has been initialized. - if self.parser: + if hasattr(self, "parser"): # TODO: wrap into non-implementation specific parser interface self.parser.aug.load() @@ -429,7 +455,7 @@ class ApacheConfigurator(common.Installer): the function is unable to correctly revert the configuration """ - super(ApacheConfigurator, self).rollback_checkpoints(rollback) + super().rollback_checkpoints(rollback) self.parser.aug.load() def _verify_exe_availability(self, exe): @@ -443,7 +469,7 @@ class ApacheConfigurator(common.Installer): """Initializes the ApacheParser""" # If user provided vhost_root value in command line, use it return parser.ApacheParser( - self.option("server_root"), self.conf("vhost-root"), + self.options.server_root, self.conf("vhost-root"), self.version, configurator=self) def get_parsernode_root(self, metadata): @@ -451,9 +477,9 @@ class ApacheConfigurator(common.Installer): if HAS_APACHECONFIG: apache_vars = {} - apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) - apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) - apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) + apache_vars["defines"] = apache_util.parse_defines(self.options.ctl) + apache_vars["includes"] = apache_util.parse_includes(self.options.ctl) + apache_vars["modules"] = apache_util.parse_modules(self.options.ctl) metadata["apache_vars"] = apache_vars with open(self.parser.loc["root"]) as f: @@ -1050,6 +1076,9 @@ class ApacheConfigurator(common.Installer): :rtype: list """ + if not self.parser_root: + raise errors.Error("This ApacheConfigurator instance is not" # pragma: no cover + " configured to use a node parser.") vhs = [] vhosts = self.parser_root.find_blocks("VirtualHost", exclude=False) for vhblock in vhosts: @@ -1302,7 +1331,7 @@ class ApacheConfigurator(common.Installer): :param boolean temp: If the change is temporary """ - if self.option("handle_modules"): + if self.options.handle_modules: if self.version >= (2, 4) and ("socache_shmcb_module" not in self.parser.modules): self.enable_mod("socache_shmcb", temp=temp) @@ -1322,7 +1351,7 @@ class ApacheConfigurator(common.Installer): Duplicates vhost and adds default ssl options New vhost will reside as (nonssl_vhost.path) + - ``self.option("le_vhost_ext")`` + ``self.options.le_vhost_ext`` .. note:: This function saves the configuration @@ -1421,15 +1450,15 @@ class ApacheConfigurator(common.Installer): """ if self.conf("vhost-root") and os.path.exists(self.conf("vhost-root")): - fp = os.path.join(filesystem.realpath(self.option("vhost_root")), + fp = os.path.join(filesystem.realpath(self.options.vhost_root), os.path.basename(non_ssl_vh_fp)) else: # Use non-ssl filepath fp = filesystem.realpath(non_ssl_vh_fp) if fp.endswith(".conf"): - return fp[:-(len(".conf"))] + self.option("le_vhost_ext") - return fp + self.option("le_vhost_ext") + return fp[:-(len(".conf"))] + self.options.le_vhost_ext + return fp + self.options.le_vhost_ext def _sift_rewrite_rule(self, line): """Decides whether a line should be copied to a SSL vhost. @@ -2273,7 +2302,7 @@ class ApacheConfigurator(common.Installer): addr in self._get_proposed_addrs(ssl_vhost)), servername, serveralias, " ".join(rewrite_rule_args), - self.option("logs_root"))) + self.options.logs_root)) def _write_out_redirect(self, ssl_vhost, text): # This is the default name @@ -2285,7 +2314,7 @@ class ApacheConfigurator(common.Installer): if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name - redirect_filepath = os.path.join(self.option("vhost_root"), + redirect_filepath = os.path.join(self.options.vhost_root, redirect_filename) # Register the new file that will be created @@ -2405,19 +2434,18 @@ class ApacheConfigurator(common.Installer): """ try: - util.run_script(self.option("restart_cmd")) + util.run_script(self.options.restart_cmd) except errors.SubprocessError as err: logger.info("Unable to restart apache using %s", - self.option("restart_cmd")) - alt_restart = self.option("restart_cmd_alt") + self.options.restart_cmd) + alt_restart = self.options.restart_cmd_alt if alt_restart: logger.debug("Trying alternative restart command: %s", alt_restart) # There is an alternative restart command available # This usually is "restart" verb while original is "graceful" try: - util.run_script(self.option( - "restart_cmd_alt")) + util.run_script(self.options.restart_cmd_alt) return except errors.SubprocessError as secerr: error = str(secerr) @@ -2432,7 +2460,7 @@ class ApacheConfigurator(common.Installer): """ try: - util.run_script(self.option("conftest_cmd")) + util.run_script(self.options.conftest_cmd) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -2448,11 +2476,11 @@ class ApacheConfigurator(common.Installer): """ try: - stdout, _ = util.run_script(self.option("version_cmd")) + stdout, _ = util.run_script(self.options.version_cmd) except errors.SubprocessError: raise errors.PluginError( "Unable to run %s -v" % - self.option("version_cmd")) + self.options.version_cmd) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) matches = regex.findall(stdout) diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 66d888e97..83a1a8e08 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -47,7 +47,7 @@ class ApacheHttp01(common.ChallengePerformer): """ def __init__(self, *args, **kwargs): - super(ApacheHttp01, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.challenge_conf_pre = os.path.join( self.configurator.conf("challenge-location"), "le_http_01_challenge_pre.conf") diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 38c21b785..106e2778a 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -238,7 +238,7 @@ class CommentNode(ParserNode, metaclass=abc.ABCMeta): created or changed after the last save. Default: False. :type dirty: bool """ - super(CommentNode, self).__init__(ancestor=kwargs['ancestor'], + super().__init__(ancestor=kwargs['ancestor'], dirty=kwargs.get('dirty', False), filepath=kwargs['filepath'], metadata=kwargs.get('metadata', {})) # pragma: no cover @@ -302,7 +302,7 @@ class DirectiveNode(ParserNode, metaclass=abc.ABCMeta): :type enabled: bool """ - super(DirectiveNode, self).__init__(ancestor=kwargs['ancestor'], + super().__init__(ancestor=kwargs['ancestor'], dirty=kwargs.get('dirty', False), filepath=kwargs['filepath'], metadata=kwargs.get('metadata', {})) # pragma: no cover diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 21fd042f6..9001a860d 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -26,7 +26,7 @@ class Addr(common.Addr): def __hash__(self): # pylint: disable=useless-super-delegation # Python 3 requires explicit overridden for __hash__ if __eq__ or # __cmp__ is overridden. See https://bugs.python.org/issue2235 - return super(Addr, self).__hash__() + return super().__hash__() def _addr_less_specific(self, addr): """Returns if addr.get_addr() is more specific than self.get_addr().""" diff --git a/certbot-apache/certbot_apache/_internal/override_arch.py b/certbot-apache/certbot_apache/_internal/override_arch.py index 1c3aed1dc..30d161a4e 100644 --- a/certbot-apache/certbot_apache/_internal/override_arch.py +++ b/certbot-apache/certbot_apache/_internal/override_arch.py @@ -3,13 +3,14 @@ import zope.interface from certbot import interfaces from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class ArchConfigurator(configurator.ApacheConfigurator): """Arch Linux specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/httpd", vhost_root="/etc/httpd/conf", vhost_files="*.conf", @@ -18,11 +19,5 @@ class ArchConfigurator(configurator.ApacheConfigurator): version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/httpd/conf", - bin=None, ) diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 54edfc911..c1a69885c 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -1,5 +1,6 @@ """ Distribution specific override class for CentOS family (RHEL, Fedora) """ import logging +from typing import cast from typing import List import zope.interface @@ -11,6 +12,7 @@ from certbot.errors import MisconfigurationError from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser +from certbot_apache._internal.configurator import OsOptions logger = logging.getLogger(__name__) @@ -19,7 +21,7 @@ logger = logging.getLogger(__name__) class CentOSConfigurator(configurator.ApacheConfigurator): """CentOS specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/httpd", vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", @@ -29,13 +31,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): restart_cmd=['apachectl', 'graceful'], restart_cmd_alt=['apachectl', 'restart'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/httpd/conf.d", - bin=None, ) def config_test(self): @@ -50,7 +46,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): fedora = os_info[0].lower() == "fedora" try: - super(CentOSConfigurator, self).config_test() + super().config_test() except errors.MisconfigurationError: if fedora: self._try_restart_fedora() @@ -68,20 +64,22 @@ class CentOSConfigurator(configurator.ApacheConfigurator): raise errors.MisconfigurationError(str(err)) # Finish with actual config check to see if systemctl restart helped - super(CentOSConfigurator, self).config_test() + super().config_test() def _prepare_options(self): """ Override the options dictionary initialization in order to support alternative restart cmd used in CentOS. """ - super(CentOSConfigurator, self)._prepare_options() - self.options["restart_cmd_alt"][0] = self.option("ctl") + super()._prepare_options() + if not self.options.restart_cmd_alt: # pragma: no cover + raise ValueError("OS option restart_cmd_alt must be set for CentOS.") + self.options.restart_cmd_alt[0] = self.options.ctl def get_parser(self): """Initializes the ApacheParser""" return CentOSParser( - self.option("server_root"), self.option("vhost_root"), + self.options.server_root, self.options.vhost_root, self.version, configurator=self) def _deploy_cert(self, *args, **kwargs): # pylint: disable=arguments-differ @@ -90,7 +88,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): has "LoadModule ssl_module..." before parsing the VirtualHost configuration that was created by Certbot """ - super(CentOSConfigurator, self)._deploy_cert(*args, **kwargs) + super()._deploy_cert(*args, **kwargs) if self.version < (2, 4, 0): self._deploy_loadmodule_ssl_if_needed() @@ -118,8 +116,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator): else: loadmod_args = path_args - if self.parser.not_modssl_ifmodule(noarg_path): # pylint: disable=no-member - if self.parser.loc["default"] in noarg_path: + centos_parser: CentOSParser = cast(CentOSParser, self.parser) + if centos_parser.not_modssl_ifmodule(noarg_path): + if centos_parser.loc["default"] in noarg_path: # LoadModule already in the main configuration file if ("ifmodule/" in noarg_path.lower() or "ifmodule[1]" in noarg_path.lower()): @@ -167,12 +166,12 @@ class CentOSParser(parser.ApacheParser): def __init__(self, *args, **kwargs): # CentOS specific configuration file for Apache self.sysconfig_filep = "/etc/sysconfig/httpd" - super(CentOSParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ # Opportunistic, works if SELinux not enforced - super(CentOSParser, self).update_runtime_variables() + super().update_runtime_variables() self.parse_sysconfig_var() def parse_sysconfig_var(self): diff --git a/certbot-apache/certbot_apache/_internal/override_darwin.py b/certbot-apache/certbot_apache/_internal/override_darwin.py index 106f5fbab..e1dca7f5e 100644 --- a/certbot-apache/certbot_apache/_internal/override_darwin.py +++ b/certbot-apache/certbot_apache/_internal/override_darwin.py @@ -3,26 +3,19 @@ import zope.interface from certbot import interfaces from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class DarwinConfigurator(configurator.ApacheConfigurator): """macOS specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( - server_root="/etc/apache2", + OS_DEFAULTS = OsOptions( vhost_root="/etc/apache2/other", vhost_files="*.conf", - logs_root="/var/log/apache2", ctl="apachectl", version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/apache2/other", - bin=None, ) diff --git a/certbot-apache/certbot_apache/_internal/override_debian.py b/certbot-apache/certbot_apache/_internal/override_debian.py index 9f938046b..7f12f4bbc 100644 --- a/certbot-apache/certbot_apache/_internal/override_debian.py +++ b/certbot-apache/certbot_apache/_internal/override_debian.py @@ -10,6 +10,7 @@ from certbot.compat import filesystem from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions logger = logging.getLogger(__name__) @@ -18,22 +19,11 @@ logger = logging.getLogger(__name__) class DebianConfigurator(configurator.ApacheConfigurator): """Debian specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/sites-available", - vhost_files="*", - logs_root="/var/log/apache2", - ctl="apache2ctl", - version_cmd=['apache2ctl', '-v'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], + OS_DEFAULTS = OsOptions( enmod="a2enmod", dismod="a2dismod", - le_vhost_ext="-le-ssl.conf", handle_modules=True, handle_sites=True, - challenge_location="/etc/apache2", - bin=None, ) def enable_site(self, vhost): @@ -58,7 +48,7 @@ class DebianConfigurator(configurator.ApacheConfigurator): if not os.path.isdir(os.path.dirname(enabled_path)): # For some reason, sites-enabled / sites-available do not exist # Call the parent method - return super(DebianConfigurator, self).enable_site(vhost) + return super().enable_site(vhost) self.reverter.register_file_creation(False, enabled_path) try: os.symlink(vhost.filep, enabled_path) @@ -132,11 +122,11 @@ class DebianConfigurator(configurator.ApacheConfigurator): # Generate reversal command. # Try to be safe here... check that we can probably reverse before # applying enmod command - if not util.exe_exists(self.option("dismod")): + if not util.exe_exists(self.options.dismod): raise errors.MisconfigurationError( "Unable to find a2dismod, please make sure a2enmod and " "a2dismod are configured correctly for certbot.") self.reverter.register_undo_command( - temp, [self.option("dismod"), "-f", mod_name]) - util.run_script([self.option("enmod"), mod_name]) + temp, [self.options.dismod, "-f", mod_name]) + util.run_script([self.options.enmod, mod_name]) diff --git a/certbot-apache/certbot_apache/_internal/override_fedora.py b/certbot-apache/certbot_apache/_internal/override_fedora.py index 9b521846c..3b947a823 100644 --- a/certbot-apache/certbot_apache/_internal/override_fedora.py +++ b/certbot-apache/certbot_apache/_internal/override_fedora.py @@ -7,13 +7,14 @@ from certbot import util from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class FedoraConfigurator(configurator.ApacheConfigurator): """Fedora 29+ specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/httpd", vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", @@ -23,13 +24,7 @@ class FedoraConfigurator(configurator.ApacheConfigurator): restart_cmd=['apachectl', 'graceful'], restart_cmd_alt=['apachectl', 'restart'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/httpd/conf.d", - bin=None, ) def config_test(self): @@ -40,14 +35,14 @@ class FedoraConfigurator(configurator.ApacheConfigurator): during the first (re)start of httpd. """ try: - super(FedoraConfigurator, self).config_test() + super().config_test() except errors.MisconfigurationError: self._try_restart_fedora() def get_parser(self): """Initializes the ApacheParser""" return FedoraParser( - self.option("server_root"), self.option("vhost_root"), + self.options.server_root, self.options.vhost_root, self.version, configurator=self) def _try_restart_fedora(self): @@ -60,7 +55,7 @@ class FedoraConfigurator(configurator.ApacheConfigurator): raise errors.MisconfigurationError(str(err)) # Finish with actual config check to see if systemctl restart helped - super(FedoraConfigurator, self).config_test() + super().config_test() def _prepare_options(self): """ @@ -68,10 +63,12 @@ class FedoraConfigurator(configurator.ApacheConfigurator): instead of httpd and so take advantages of this new bash script in newer versions of Fedora to restart httpd. """ - super(FedoraConfigurator, self)._prepare_options() - self.options["restart_cmd"][0] = 'apachectl' - self.options["restart_cmd_alt"][0] = 'apachectl' - self.options["conftest_cmd"][0] = 'apachectl' + super()._prepare_options() + self.options.restart_cmd[0] = 'apachectl' + if not self.options.restart_cmd_alt: # pragma: no cover + raise ValueError("OS option restart_cmd_alt must be set for Fedora.") + self.options.restart_cmd_alt[0] = 'apachectl' + self.options.conftest_cmd[0] = 'apachectl' class FedoraParser(parser.ApacheParser): @@ -79,12 +76,12 @@ class FedoraParser(parser.ApacheParser): def __init__(self, *args, **kwargs): # Fedora 29+ specific configuration file for Apache self.sysconfig_filep = "/etc/sysconfig/httpd" - super(FedoraParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ # Opportunistic, works if SELinux not enforced - super(FedoraParser, self).update_runtime_variables() + super().update_runtime_variables() self._parse_sysconfig_var() def _parse_sysconfig_var(self): diff --git a/certbot-apache/certbot_apache/_internal/override_gentoo.py b/certbot-apache/certbot_apache/_internal/override_gentoo.py index 59a3d981f..1b86c925e 100644 --- a/certbot-apache/certbot_apache/_internal/override_gentoo.py +++ b/certbot-apache/certbot_apache/_internal/override_gentoo.py @@ -5,29 +5,19 @@ from certbot import interfaces from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class GentooConfigurator(configurator.ApacheConfigurator): """Gentoo specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/apache2", vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", - logs_root="/var/log/apache2", - ctl="apache2ctl", - version_cmd=['apache2ctl', '-v'], - restart_cmd=['apache2ctl', 'graceful'], restart_cmd_alt=['apache2ctl', 'restart'], - conftest_cmd=['apache2ctl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - bin=None, ) def _prepare_options(self): @@ -35,13 +25,15 @@ class GentooConfigurator(configurator.ApacheConfigurator): Override the options dictionary initialization in order to support alternative restart cmd used in Gentoo. """ - super(GentooConfigurator, self)._prepare_options() - self.options["restart_cmd_alt"][0] = self.option("ctl") + super()._prepare_options() + if not self.options.restart_cmd_alt: # pragma: no cover + raise ValueError("OS option restart_cmd_alt must be set for Gentoo.") + self.options.restart_cmd_alt[0] = self.options.ctl def get_parser(self): """Initializes the ApacheParser""" return GentooParser( - self.option("server_root"), self.option("vhost_root"), + self.options.server_root, self.options.vhost_root, self.version, configurator=self) @@ -50,7 +42,7 @@ class GentooParser(parser.ApacheParser): def __init__(self, *args, **kwargs): # Gentoo specific configuration file for Apache2 self.apacheconfig_filep = "/etc/conf.d/apache2" - super(GentooParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ @@ -66,7 +58,7 @@ class GentooParser(parser.ApacheParser): def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.option("ctl"), "modules"] + mod_cmd = [self.configurator.options.ctl, "modules"] matches = apache_util.parse_from_subprocess(mod_cmd, r"(.*)_module") for mod in matches: self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/_internal/override_suse.py b/certbot-apache/certbot_apache/_internal/override_suse.py index afce98dfa..d692fd239 100644 --- a/certbot-apache/certbot_apache/_internal/override_suse.py +++ b/certbot-apache/certbot_apache/_internal/override_suse.py @@ -3,26 +3,21 @@ import zope.interface from certbot import interfaces from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class OpenSUSEConfigurator(configurator.ApacheConfigurator): """OpenSUSE specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( - server_root="/etc/apache2", + OS_DEFAULTS = OsOptions( vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", - logs_root="/var/log/apache2", ctl="apachectl", version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], enmod="a2enmod", dismod="a2dismod", - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - bin=None, ) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 1a6af4c4b..141991ccc 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -5,12 +5,18 @@ import logging import re from typing import Dict from typing import List +from typing import Optional from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import constants +try: + from augeas import Augeas +except ImportError: # pragma: no cover + Augeas = None # type: ignore + logger = logging.getLogger(__name__) @@ -39,8 +45,7 @@ class ApacheParser: self.configurator = configurator # Initialize augeas - self.aug = None - self.init_augeas() + self.aug = init_augeas() if not self.check_aug_version(): raise errors.NotSupportedError( @@ -48,7 +53,7 @@ class ApacheParser: "version 1.2.0 or higher, please make sure you have you have " "those installed.") - self.modules: Dict[str, str] = {} + self.modules: Dict[str, Optional[str]] = {} self.parser_paths: Dict[str, List[str]] = {} self.variables: Dict[str, str] = {} @@ -76,30 +81,13 @@ class ApacheParser: # Must also attempt to parse additional virtual host root if vhostroot: self.parse_file(os.path.abspath(vhostroot) + "/" + - self.configurator.option("vhost_files")) + self.configurator.options.vhost_files) # check to see if there were unparsed define statements if version < (2, 4): if self.find_dir("Define", exclude=False): raise errors.PluginError("Error parsing runtime variables") - def init_augeas(self): - """ Initialize the actual Augeas instance """ - - try: - import augeas - except ImportError: # pragma: no cover - raise errors.NoInstallationError("Problem in Augeas installation") - - self.aug = augeas.Augeas( - # specify a directory to load our preferred lens from - loadpath=constants.AUGEAS_LENS_DIR, - # Do not save backup (we do it ourselves), do not load - # anything by default - flags=(augeas.Augeas.NONE | - augeas.Augeas.NO_MODL_AUTOLOAD | - augeas.Augeas.ENABLE_SPAN)) - def check_parsing_errors(self, lens): """Verify Augeas can parse all of the lens files. @@ -294,7 +282,7 @@ class ApacheParser: def update_defines(self): """Updates the dictionary of known variables in the configuration""" - self.variables = apache_util.parse_defines(self.configurator.option("ctl")) + self.variables = apache_util.parse_defines(self.configurator.options.ctl) def update_includes(self): """Get includes from httpd process, and add them to DOM if needed""" @@ -304,7 +292,7 @@ class ApacheParser: # configuration files _ = self.find_dir("Include") - matches = apache_util.parse_includes(self.configurator.option("ctl")) + matches = apache_util.parse_includes(self.configurator.options.ctl) if matches: for i in matches: if not self.parsed_in_current(i): @@ -313,7 +301,7 @@ class ApacheParser: def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - matches = apache_util.parse_modules(self.configurator.option("ctl")) + matches = apache_util.parse_modules(self.configurator.options.ctl) for mod in matches: self.add_mod(mod.strip()) @@ -949,3 +937,19 @@ def get_aug_path(file_path): """ return "/files%s" % file_path + + +def init_augeas() -> Augeas: + """ Initialize the actual Augeas instance """ + + if not Augeas: # pragma: no cover + raise errors.NoInstallationError("Problem in Augeas installation") + + return Augeas( + # specify a directory to load our preferred lens from + loadpath=constants.AUGEAS_LENS_DIR, + # Do not save backup (we do it ourselves), do not load + # anything by default + flags=(Augeas.NONE | + Augeas.NO_MODL_AUTOLOAD | + Augeas.ENABLE_SPAN)) diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 43165a9dd..898e4e3e7 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -24,7 +24,7 @@ setup( description="Apache plugin for Certbot", url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 83224fc98..85a17ab31 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -1,4 +1,6 @@ """Tests for AugeasParserNode classes""" +from typing import List + try: import mock except ImportError: # pragma: no cover @@ -27,7 +29,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- """Test AugeasParserNode using available test configurations""" def setUp(self): # pylint: disable=arguments-differ - super(AugeasParserNodeTest, self).setUp() + super().setUp() with mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.get_parsernode_root") as mock_parsernode: mock_parsernode.side_effect = _get_augeasnode_mock( diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index db1c46b52..8c8ba4873 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -18,7 +18,7 @@ class AutoHSTSTest(util.ApacheTest): # pylint: disable=protected-access def setUp(self): # pylint: disable=arguments-differ - super(AutoHSTSTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) diff --git a/certbot-apache/tests/centos6_test.py b/certbot-apache/tests/centos6_test.py index 27b4f8e80..bfbe22ad0 100644 --- a/certbot-apache/tests/centos6_test.py +++ b/certbot-apache/tests/centos6_test.py @@ -36,9 +36,9 @@ class CentOS6Tests(util.ApacheTest): test_dir = "centos6_apache/apache" config_root = "centos6_apache/apache/httpd" vhost_root = "centos6_apache/apache/httpd/conf.d" - super(CentOS6Tests, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, diff --git a/certbot-apache/tests/centos_test.py b/certbot-apache/tests/centos_test.py index b7e9c1cb6..a92c37979 100644 --- a/certbot-apache/tests/centos_test.py +++ b/certbot-apache/tests/centos_test.py @@ -41,9 +41,9 @@ class FedoraRestartTest(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(FedoraRestartTest, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="fedora_old") @@ -96,9 +96,9 @@ class MultipleVhostsTestCentOS(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(MultipleVhostsTestCentOS, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, diff --git a/certbot-apache/tests/complex_parsing_test.py b/certbot-apache/tests/complex_parsing_test.py index 8b795b0b6..e36bd85d1 100644 --- a/certbot-apache/tests/complex_parsing_test.py +++ b/certbot-apache/tests/complex_parsing_test.py @@ -11,7 +11,7 @@ class ComplexParserTest(util.ParserTest): """Apache Parser Test.""" def setUp(self): # pylint: disable=arguments-differ - super(ComplexParserTest, self).setUp( + super().setUp( "complex_parsing", "complex_parsing") self.setup_variables() diff --git a/certbot-apache/tests/configurator_reverter_test.py b/certbot-apache/tests/configurator_reverter_test.py index 8596195d8..d8f5ddd05 100644 --- a/certbot-apache/tests/configurator_reverter_test.py +++ b/certbot-apache/tests/configurator_reverter_test.py @@ -16,7 +16,7 @@ class ConfiguratorReverterTest(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ - super(ConfiguratorReverterTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 302bf0023..8620955aa 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -30,7 +30,7 @@ class MultipleVhostsTest(util.ApacheTest): """Test two standard well-configured HTTP vhosts.""" def setUp(self): # pylint: disable=arguments-differ - super(MultipleVhostsTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) @@ -103,9 +103,9 @@ class MultipleVhostsTest(util.ApacheTest): "handle_modules", "handle_sites", "ctl"] exp = {} - for k in ApacheConfigurator.OS_DEFAULTS: + for k in ApacheConfigurator.OS_DEFAULTS.__dict__.keys(): if k in parserargs: - exp[k.replace("_", "-")] = ApacheConfigurator.OS_DEFAULTS[k] + exp[k.replace("_", "-")] = getattr(ApacheConfigurator.OS_DEFAULTS, k) # Special cases exp["vhost-root"] = None @@ -128,14 +128,13 @@ class MultipleVhostsTest(util.ApacheTest): def test_all_configurators_defaults_defined(self): from certbot_apache._internal.entrypoint import OVERRIDE_CLASSES from certbot_apache._internal.configurator import ApacheConfigurator - parameters = set(ApacheConfigurator.OS_DEFAULTS.keys()) + parameters = set(ApacheConfigurator.OS_DEFAULTS.__dict__.keys()) for cls in OVERRIDE_CLASSES.values(): - self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.keys()))) + self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.__dict__.keys()))) def test_constant(self): self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in - self.config.option("server_root")) - self.assertEqual(self.config.option("nonexistent"), None) + self.config.options.server_root) @certbot_util.patch_get_utility() def test_get_all_names(self, mock_getutility): @@ -1477,9 +1476,9 @@ class AugeasVhostsTest(util.ApacheTest): td = "debian_apache_2_4/augeas_vhosts" cr = "debian_apache_2_4/augeas_vhosts/apache2" vr = "debian_apache_2_4/augeas_vhosts/apache2/sites-available" - super(AugeasVhostsTest, self).setUp(test_dir=td, - config_root=cr, - vhost_root=vr) + super().setUp(test_dir=td, + config_root=cr, + vhost_root=vr) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, @@ -1556,9 +1555,9 @@ class MultiVhostsTest(util.ApacheTest): td = "debian_apache_2_4/multi_vhosts" cr = "debian_apache_2_4/multi_vhosts/apache2" vr = "debian_apache_2_4/multi_vhosts/apache2/sites-available" - super(MultiVhostsTest, self).setUp(test_dir=td, - config_root=cr, - vhost_root=vr) + super().setUp(test_dir=td, + config_root=cr, + vhost_root=vr) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, @@ -1661,7 +1660,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" def setUp(self): # pylint: disable=arguments-differ - super(InstallSslOptionsConfTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) @@ -1774,7 +1773,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): # ssl_module statically linked self.config._openssl_version = None self.config.parser.modules['ssl_module'] = None - self.config.options['bin'] = '/fake/path/to/httpd' + self.config.options.bin = '/fake/path/to/httpd' with mock.patch("certbot_apache._internal.configurator." "ApacheConfigurator._open_module_file") as mock_omf: mock_omf.return_value = some_string_contents @@ -1810,7 +1809,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): # When ssl_module is statically linked but --apache-bin not provided self.config._openssl_version = None - self.config.options['bin'] = None + self.config.options.bin = None self.config.parser.modules['ssl_module'] = None with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: self.assertEqual(self.config.openssl_version(), None) diff --git a/certbot-apache/tests/debian_test.py b/certbot-apache/tests/debian_test.py index 192e3cba5..c72b8b6ae 100644 --- a/certbot-apache/tests/debian_test.py +++ b/certbot-apache/tests/debian_test.py @@ -20,7 +20,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): _multiprocess_can_split_ = True def setUp(self): # pylint: disable=arguments-differ - super(MultipleVhostsTestDebian, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="debian") diff --git a/certbot-apache/tests/fedora_test.py b/certbot-apache/tests/fedora_test.py index 50831802b..d48559cee 100644 --- a/certbot-apache/tests/fedora_test.py +++ b/certbot-apache/tests/fedora_test.py @@ -46,9 +46,9 @@ class FedoraRestartTest(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(FedoraRestartTest, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="fedora") @@ -90,9 +90,9 @@ class MultipleVhostsTestFedora(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(MultipleVhostsTestFedora, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, diff --git a/certbot-apache/tests/gentoo_test.py b/certbot-apache/tests/gentoo_test.py index 64f7d1062..cf39ff5cb 100644 --- a/certbot-apache/tests/gentoo_test.py +++ b/certbot-apache/tests/gentoo_test.py @@ -50,9 +50,9 @@ class MultipleVhostsTestGentoo(util.ApacheTest): test_dir = "gentoo_apache/apache" config_root = "gentoo_apache/apache/apache2" vhost_root = "gentoo_apache/apache/apache2/vhosts.d" - super(MultipleVhostsTestGentoo, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) # pylint: disable=line-too-long with mock.patch("certbot_apache._internal.override_gentoo.GentooParser.update_runtime_variables"): diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 06e19e445..71f2db500 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -1,6 +1,7 @@ """Test for certbot_apache._internal.http_01.""" import unittest import errno +from typing import List try: import mock @@ -23,7 +24,7 @@ class ApacheHttp01Test(util.ApacheTest): """Test for certbot_apache._internal.http_01.ApacheHttp01.""" def setUp(self, *args, **kwargs): # pylint: disable=arguments-differ - super(ApacheHttp01Test, self).setUp(*args, **kwargs) + super().setUp(*args, **kwargs) self.account_key = self.rsa512jwk self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = [] diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py index 7aedec31d..e30eca153 100644 --- a/certbot-apache/tests/parser_test.py +++ b/certbot-apache/tests/parser_test.py @@ -16,7 +16,7 @@ class BasicParserTest(util.ParserTest): """Apache Parser Test.""" def setUp(self): # pylint: disable=arguments-differ - super(BasicParserTest, self).setUp() + super().setUp() def tearDown(self): shutil.rmtree(self.temp_dir) @@ -305,11 +305,9 @@ class BasicParserTest(util.ParserTest): self.assertRaises( errors.PluginError, self.parser.update_runtime_variables) - @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.option") @mock.patch("certbot_apache._internal.apache_util.subprocess.Popen") - def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_opt): + def test_update_runtime_vars_bad_ctl(self, mock_popen): mock_popen.side_effect = OSError - mock_opt.return_value = "nonexistent" self.assertRaises( errors.MisconfigurationError, self.parser.update_runtime_variables) @@ -332,14 +330,14 @@ class BasicParserTest(util.ParserTest): class ParserInitTest(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ - super(ParserInitTest, self).setUp() + super().setUp() def tearDown(self): shutil.rmtree(self.temp_dir) shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) - @mock.patch("certbot_apache._internal.parser.ApacheParser.init_augeas") + @mock.patch("certbot_apache._internal.parser.init_augeas") def test_prepare_no_augeas(self, mock_init_augeas): from certbot_apache._internal.parser import ApacheParser mock_init_augeas.side_effect = errors.NoInstallationError diff --git a/certbot-apache/tests/parsernode_configurator_test.py b/certbot-apache/tests/parsernode_configurator_test.py index 7fbec2540..411871a43 100644 --- a/certbot-apache/tests/parsernode_configurator_test.py +++ b/certbot-apache/tests/parsernode_configurator_test.py @@ -20,7 +20,7 @@ class ConfiguratorParserNodeTest(util.ApacheTest): # pylint: disable=too-many-p """Test AugeasParserNode using available test configurations""" def setUp(self): # pylint: disable=arguments-differ - super(ConfiguratorParserNodeTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, diff --git a/certbot-apache/tests/parsernode_test.py b/certbot-apache/tests/parsernode_test.py index a86952f53..4ea5f8415 100644 --- a/certbot-apache/tests/parsernode_test.py +++ b/certbot-apache/tests/parsernode_test.py @@ -18,7 +18,7 @@ class DummyParserNode(interfaces.ParserNode): self.dirty = dirty self.filepath = filepath self.metadata = metadata - super(DummyParserNode, self).__init__(**kwargs) + super().__init__(**kwargs) def save(self, msg): # pragma: no cover """Save""" @@ -38,7 +38,7 @@ class DummyCommentNode(DummyParserNode): """ comment, kwargs = util.commentnode_kwargs(kwargs) self.comment = comment - super(DummyCommentNode, self).__init__(**kwargs) + super().__init__(**kwargs) class DummyDirectiveNode(DummyParserNode): @@ -54,7 +54,7 @@ class DummyDirectiveNode(DummyParserNode): self.parameters = parameters self.enabled = enabled - super(DummyDirectiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) def set_parameters(self, parameters): # pragma: no cover """Set parameters""" diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index 18c7e5aca..a0b44d188 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -67,7 +67,7 @@ class ParserTest(ApacheTest): def setUp(self, test_dir="debian_apache_2_4/multiple_vhosts", config_root="debian_apache_2_4/multiple_vhosts/apache2", vhost_root="debian_apache_2_4/multiple_vhosts/apache2/sites-available"): - super(ParserTest, self).setUp(test_dir, config_root, vhost_root) + super().setUp(test_dir, config_root, vhost_root) zope.component.provideUtility(display_util.FileDisplay(sys.stdout, False)) @@ -123,11 +123,11 @@ def get_apache_configurator( version=version, use_parsernode=use_parsernode, openssl_version=openssl_version) if not conf_vhost_path: - config_class.OS_DEFAULTS["vhost_root"] = vhost_path + config_class.OS_DEFAULTS.vhost_root = vhost_path else: # Custom virtualhost path was requested config.config.apache_vhost_root = conf_vhost_path - config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"] + config.config.apache_ctl = config_class.OS_DEFAULTS.ctl config.prepare() return config diff --git a/certbot-auto b/certbot-auto index 24af0b4c3..c37c45596 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.13.0" +LE_AUTO_VERSION="1.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -800,6 +800,7 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 @@ -1488,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.13.0 \ - --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ - --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 -acme==1.13.0 \ - --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ - --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 -certbot-apache==1.13.0 \ - --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ - --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 -certbot-nginx==1.13.0 \ - --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ - --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 +certbot==1.14.0 \ + --hash=sha256:67b4d26ceaea6c7f8325d0d45169e7a165a2cabc7122c84bc971ba068ca19cca \ + --hash=sha256:959ea90c6bb8dca38eab9772722cb940972ef6afcd5f15deef08b3c3636841eb +acme==1.14.0 \ + --hash=sha256:4f48c41261202f1a389ec2986b2580b58f53e0d5a1ae2463b34318d78b87fc66 \ + --hash=sha256:61daccfb0343628cbbca551a7fc4c82482113952c21db3fe0c585b7c98fa1c35 +certbot-apache==1.14.0 \ + --hash=sha256:b757038db23db707c44630fecb46e99172bd791f0db5a8e623c0842613c4d3d9 \ + --hash=sha256:887fe4a21af2de1e5c2c9428bacba6eb7c1219257bc70f1a1d8447c8a321adb0 +certbot-nginx==1.14.0 \ + --hash=sha256:8916a815437988d6c192df9f035bb7a176eab20eee0956677b335d0698d243fb \ + --hash=sha256:cc2a8a0de56d9bb6b2efbda6c80c647dad8db2bb90675cac03ade94bd5fc8597 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index b052e375d..9af1bc15c 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -12,9 +12,9 @@ class IntegrationTestsContext: def __init__(self, request): self.request = request - if hasattr(request.config, 'slaveinput'): # Worker node - self.worker_id = request.config.slaveinput['slaveid'] - acme_xdist = request.config.slaveinput['acme_xdist'] + if hasattr(request.config, 'workerinput'): # Worker node + self.worker_id = request.config.workerinput['workerid'] + acme_xdist = request.config.workerinput['acme_xdist'] else: # Primary node self.worker_id = 'primary' acme_xdist = request.config.acme_xdist diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 0b649acdc..c9fa5ff65 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -8,19 +8,20 @@ import shutil import subprocess import time -from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1, SECP384R1, SECP521R1 +from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1 +from cryptography.hazmat.primitives.asymmetric.ec import SECP384R1 +from cryptography.hazmat.primitives.asymmetric.ec import SECP521R1 from cryptography.x509 import NameOID - import pytest from certbot_integration_tests.certbot_tests import context as certbot_context from certbot_integration_tests.certbot_tests.assertions import assert_cert_count_for_lineage from certbot_integration_tests.certbot_tests.assertions import assert_elliptic_key -from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_owner from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_permissions from certbot_integration_tests.certbot_tests.assertions import assert_equals_world_read_permissions from certbot_integration_tests.certbot_tests.assertions import assert_hook_execution +from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key from certbot_integration_tests.certbot_tests.assertions import assert_saved_renew_hook from certbot_integration_tests.certbot_tests.assertions import assert_world_no_permissions from certbot_integration_tests.certbot_tests.assertions import assert_world_read_permissions diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index 6fc77480d..5e6ed5562 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -34,7 +34,7 @@ def pytest_configure(config): Standard pytest hook used to add a configuration logic for each node of a pytest run. :param config: the current pytest configuration """ - if not hasattr(config, 'slaveinput'): # If true, this is the primary node + if not hasattr(config, 'workerinput'): # If true, this is the primary node with _print_on_err(): _setup_primary_node(config) @@ -44,8 +44,8 @@ def pytest_configure_node(node): Standard pytest-xdist hook used to configure a worker node. :param node: current worker node """ - node.slaveinput['acme_xdist'] = node.config.acme_xdist - node.slaveinput['dns_xdist'] = node.config.dns_xdist + node.workerinput['acme_xdist'] = node.config.acme_xdist + node.workerinput['dns_xdist'] = node.config.dns_xdist @contextlib.contextmanager diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/context.py b/certbot-ci/certbot_integration_tests/nginx_tests/context.py index 6f0f833a0..3ee1766a1 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/context.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/context.py @@ -11,7 +11,7 @@ from certbot_integration_tests.utils import misc class IntegrationTestsContext(certbot_context.IntegrationTestsContext): """General fixture describing a certbot-nginx integration tests context""" def __init__(self, request): - super(IntegrationTestsContext, self).__init__(request) + super().__init__(request) self.nginx_root = os.path.join(self.workspace, 'nginx') os.mkdir(self.nginx_root) @@ -29,7 +29,7 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): def cleanup(self): self._stop_nginx() - super(IntegrationTestsContext, self).cleanup() + super().cleanup() def certbot_test_nginx(self, args): """ diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py index c8024b21d..3d4147313 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -13,13 +13,12 @@ from certbot_integration_tests.utils import certbot_call class IntegrationTestsContext(certbot_context.IntegrationTestsContext): """Integration test context for certbot-dns-rfc2136""" def __init__(self, request): - super(IntegrationTestsContext, self).__init__(request) + super().__init__(request) self.request = request - self._dns_xdist = None - if hasattr(request.config, 'slaveinput'): # Worker node - self._dns_xdist = request.config.slaveinput['dns_xdist'] + if hasattr(request.config, 'workerinput'): # Worker node + self._dns_xdist = request.config.workerinput['dns_xdist'] else: # Primary node self._dns_xdist = request.config.dns_xdist @@ -45,7 +44,6 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): src_file = resource_filename('certbot_integration_tests', 'assets/bind-config/rfc2136-credentials-{}.ini.tpl' .format(label)) - contents = None with open(src_file, 'r') as f: contents = f.read().format( diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py index b319eca4c..965ab6881 100755 --- a/certbot-ci/certbot_integration_tests/utils/certbot_call.py +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -1,10 +1,10 @@ #!/usr/bin/env python """Module to call certbot in test mode""" +from distutils.version import LooseVersion import os import subprocess import sys -from distutils.version import LooseVersion import certbot_integration_tests # pylint: disable=wildcard-import,unused-wildcard-import diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index 31e7aee18..48f5a533e 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -8,6 +8,7 @@ import subprocess import sys import tempfile import time +from typing import Optional from pkg_resources import resource_filename @@ -38,7 +39,7 @@ class DNSServer: self.bind_root = tempfile.mkdtemp() - self.process: subprocess.Popen = None + self.process: Optional[subprocess.Popen] = None self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]} @@ -119,6 +120,9 @@ class DNSServer: but otherwise the contents are ignored. :param int attempts: The number of attempts to make. """ + if not self.process: + raise ValueError("DNS server has not been started. Please run start() first.") + for _ in range(attempts): if self.process.poll(): raise ValueError("BIND9 server stopped unexpectedly") diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py index 4db72998f..3458929ad 100755 --- a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py @@ -29,10 +29,7 @@ class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediates/0', verify=False) issuer_cert = x509.load_pem_x509_certificate(request.content, default_backend()) - try: - content_len = int(self.headers.getheader('content-length', 0)) - except AttributeError: - content_len = int(self.headers.get('Content-Length')) + content_len = int(self.headers.get('Content-Length')) ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len)) response = requests.get('{0}/cert-status-by-serial/{1}'.format( diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index 9f9c1f462..9d52b6268 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -7,6 +7,13 @@ from setuptools import setup version = '0.32.0.dev0' +# setuptools 36.2+ is needed for support for environment markers +min_setuptools_version='36.2' +# This conditional isn't necessary, but it provides better error messages to +# people who try to install this package with older versions of setuptools. +if LooseVersion(setuptools_version) < LooseVersion(min_setuptools_version): + raise RuntimeError(f'setuptools {min_setuptools_version}+ is required') + install_requires = [ 'coverage', 'cryptography', @@ -14,30 +21,24 @@ install_requires = [ 'pyopenssl', 'pytest', 'pytest-cov', - 'pytest-xdist', + # This version is needed for "worker" attributes we currently use like + # "workerinput". See https://github.com/pytest-dev/pytest-xdist/pull/268. + 'pytest-xdist>=1.22.1', 'python-dateutil', + # This dependency needs to be added using environment markers to avoid its + # installation on Linux. + 'pywin32>=300 ; sys_platform == "win32"', 'pyyaml', 'requests', ] -# Add pywin32 on Windows platforms to handle low-level system calls. -# This dependency needs to be added using environment markers to avoid its installation on Linux. -# However environment markers are supported only with setuptools >= 36.2. -# So this dependency is not added for old Linux distributions with old setuptools, -# in order to allow these systems to build certbot from sources. -if LooseVersion(setuptools_version) >= LooseVersion('36.2'): - install_requires.append("pywin32>=224 ; sys_platform == 'win32'") -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') - setup( name='certbot-ci', version=version, description="Certbot continuous integration framework", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-ci/snap_integration_tests/dns_tests/test_main.py b/certbot-ci/snap_integration_tests/dns_tests/test_main.py index 721352c04..d008efc67 100644 --- a/certbot-ci/snap_integration_tests/dns_tests/test_main.py +++ b/certbot-ci/snap_integration_tests/dns_tests/test_main.py @@ -44,4 +44,4 @@ def test_dns_plugin_install(dns_snap_path): 'certbot:certbot-metadata']) subprocess.check_call(['snap', 'install', '--dangerous', dns_snap_path]) finally: - subprocess.call(['snap', 'remove', 'plugin_name']) + subprocess.call(['snap', 'remove', plugin_name]) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index 2b3f94581..909a9d37c 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -22,7 +22,7 @@ class Proxy(configurators_common.Proxy): def __init__(self, args): """Initializes the plugin with the given command line args""" - super(Proxy, self).__init__(args) + super().__init__(args) self.le_config.apache_le_vhost_ext = "-le-ssl.conf" self.modules = self.server_root = self.test_conf = self.version = None @@ -34,7 +34,7 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" - config = super(Proxy, self).load_config() + config = super().load_config() self._all_names, self._test_names = _get_names(config) server_root = _get_server_root(config) @@ -54,9 +54,9 @@ class Proxy(configurators_common.Proxy): def _prepare_configurator(self): """Prepares the Apache plugin for testing""" - for k in entrypoint.ENTRYPOINT.OS_DEFAULTS: + for k in entrypoint.ENTRYPOINT.OS_DEFAULTS.__dict__.keys(): setattr(self.le_config, "apache_" + k, - entrypoint.ENTRYPOINT.OS_DEFAULTS[k]) + getattr(entrypoint.ENTRYPOINT.OS_DEFAULTS, k)) self._configurator = entrypoint.ENTRYPOINT( config=configuration.NamespaceConfig(self.le_config), @@ -65,7 +65,7 @@ class Proxy(configurators_common.Proxy): def cleanup_from_tests(self): """Performs any necessary cleanup from running plugin tests""" - super(Proxy, self).cleanup_from_tests() + super().cleanup_from_tests() mock.patch.stopall() diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index bf768f8f8..bf26291ea 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -33,7 +33,9 @@ class Proxy: self.args = args self.http_port = 80 self.https_port = 443 - self._configurator = self._all_names = self._test_names = None + self._configurator = None + self._all_names = None + self._test_names = None def __getattr__(self, name): """Wraps the configurator methods""" @@ -93,5 +95,7 @@ class Proxy: """Installs cert""" cert_path, key_path, chain_path = self.copy_certs_and_keys( cert_path, key_path, chain_path) + if not self._configurator: + raise ValueError("Configurator plugin is not set.") self._configurator.deploy_cert( domain, cert_path, key_path, chain_path, fullchain_path) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py index d7b7a120e..e209480e3 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py @@ -21,7 +21,7 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" - config = super(Proxy, self).load_config() + config = super().load_config() self._all_names, self._test_names = _get_names(config) server_root = _get_server_root(config) diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 3e9e6a7c2..9567bbef6 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' install_requires = [ 'certbot', @@ -24,7 +24,7 @@ setup( description="Compatibility tests for Certbot", url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index 034d7bbb2..8f42b3ce9 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -3,6 +3,7 @@ import logging from typing import Any from typing import Dict from typing import List +from typing import Optional import CloudFlare import zope.interface @@ -10,6 +11,7 @@ import zope.interface from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -29,12 +31,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 120 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add) + super().add_parser_arguments(add) add('credentials', help='Cloudflare credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -79,6 +81,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_cloudflare_client().del_txt_record(domain, validation_name, validation) def _get_cloudflare_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") if self.credentials.conf('api-token'): return _CloudflareClient(None, self.credentials.conf('api-token')) return _CloudflareClient(self.credentials.conf('email'), self.credentials.conf('api-key')) diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 2e5215f9c..3e3a3d1ba 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="Cloudflare DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py index e9adf4ed9..d7075a84d 100644 --- a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py +++ b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py @@ -27,7 +27,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def setUp(self): from certbot_dns_cloudflare._internal.dns_cloudflare import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write({"cloudflare_email": EMAIL, "cloudflare_api_key": API_KEY}, path) diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py index 654c04c70..1ec5cb5ab 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py @@ -1,5 +1,6 @@ """DNS Authenticator for CloudXNS DNS.""" import logging +from typing import Optional from lexicon.providers import cloudxns import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='CloudXNS credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -56,6 +58,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_cloudxns_client().del_txt_record(domain, validation_name, validation) def _get_cloudxns_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _CloudXNSLexiconClient(self.credentials.conf('api-key'), self.credentials.conf('secret-key'), self.ttl) @@ -67,7 +71,7 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, secret_key, ttl): - super(_CloudXNSLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('cloudxns', { 'ttl': ttl, diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 44bada6bc..153e373bd 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,12 +4,12 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] @@ -36,7 +36,7 @@ setup( description="CloudXNS DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py index 43c69790f..81dea5ca4 100644 --- a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py +++ b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py @@ -26,7 +26,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_cloudxns._internal.dns_cloudxns import Authenticator diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index cb5012fb7..23b669847 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -1,5 +1,6 @@ """DNS Authenticator for DigitalOcean.""" import logging +from typing import Optional import digitalocean import zope.interface @@ -7,6 +8,7 @@ import zope.interface from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -24,12 +26,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 30 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add) + super().add_parser_arguments(add) add('credentials', help='DigitalOcean credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -53,6 +55,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_digitalocean_client().del_txt_record(domain, validation_name, validation) def _get_digitalocean_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _DigitalOceanClient(self.credentials.conf('token')) diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 106eb3cac..fba2fbc5f 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="DigitalOcean DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py index 84bb35b1f..6088262ee 100644 --- a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py @@ -23,7 +23,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def setUp(self): from certbot_dns_digitalocean._internal.dns_digitalocean import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write({"digitalocean_token": TOKEN}, path) diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py index 9f7f100d7..858ee8925 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py @@ -1,5 +1,6 @@ """DNS Authenticator for DNSimple DNS.""" import logging +from typing import Optional from lexicon.providers import dnsimple import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='DNSimple credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -54,6 +56,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_dnsimple_client().del_txt_record(domain, validation_name, validation) def _get_dnsimple_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _DNSimpleLexiconClient(self.credentials.conf('token'), self.ttl) @@ -63,7 +67,7 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, token, ttl): - super(_DNSimpleLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('dnssimple', { 'ttl': ttl, diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 99a3c92a3..70181ae78 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -32,7 +32,7 @@ if os.environ.get('SNAP_BUILD'): # which allows us to potentially upgrade our packages in these distros # as necessary. if os.environ.get('CERTBOT_OLDEST') == '1': - install_requires.append('dns-lexicon>=2.2.1') + install_requires.append('dns-lexicon>=3.1.0') # Changed parameter name else: install_requires.append('dns-lexicon>=3.2.1') @@ -47,7 +47,7 @@ setup( description="DNSimple DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py index 40eba4754..fc3dc5b1f 100644 --- a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py +++ b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py @@ -20,7 +20,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_dnsimple._internal.dns_dnsimple import Authenticator diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py index 4a1fcffc3..67903e19d 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py @@ -1,5 +1,6 @@ """DNS Authenticator for DNS Made Easy DNS.""" import logging +from typing import Optional from lexicon.providers import dnsmadeeasy import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) + super().add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help='DNS Made Easy credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -58,6 +60,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_dnsmadeeasy_client().del_txt_record(domain, validation_name, validation) def _get_dnsmadeeasy_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _DNSMadeEasyLexiconClient(self.credentials.conf('api-key'), self.credentials.conf('secret-key'), self.ttl) @@ -69,7 +73,7 @@ class _DNSMadeEasyLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, secret_key, ttl): - super(_DNSMadeEasyLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('dnsmadeeasy', { 'ttl': ttl, diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index b126b4e8f..852eac606 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,12 +4,12 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] @@ -36,7 +36,7 @@ setup( description="DNS Made Easy DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py index 4a69e977c..a04716d95 100644 --- a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py +++ b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py @@ -22,7 +22,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_dnsmadeeasy._internal.dns_dnsmadeeasy import Authenticator diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py index 39deddae5..57ff01671 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py @@ -1,12 +1,15 @@ """DNS Authenticator for Gehirn Infrastructure Service DNS.""" import logging +from typing import Optional from lexicon.providers import gehirn import zope.interface +from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='Gehirn Infrastructure Service credentials file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -57,6 +60,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_gehirn_client().del_txt_record(domain, validation_name, validation) def _get_gehirn_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _GehirnLexiconClient( self.credentials.conf('api-token'), self.credentials.conf('api-secret'), @@ -70,7 +75,7 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_token, api_secret, ttl): - super(_GehirnLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('gehirn', { 'ttl': ttl, @@ -84,4 +89,4 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return None # Expected errors when zone name guess is wrong - return super(_GehirnLexiconClient, self)._handle_http_error(e, domain_name) + return super()._handle_http_error(e, domain_name) diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index cada7a85e..85058ab90 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,11 +4,11 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'dns-lexicon>=2.1.22', + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] @@ -35,7 +35,7 @@ setup( description="Gehirn Infrastructure Service DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-gehirn/tests/dns_gehirn_test.py b/certbot-dns-gehirn/tests/dns_gehirn_test.py index 0598a5eb5..1310f74ca 100644 --- a/certbot-dns-gehirn/tests/dns_gehirn_test.py +++ b/certbot-dns-gehirn/tests/dns_gehirn_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_gehirn._internal.dns_gehirn import Authenticator diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py index 2d448c590..67ed34a45 100644 --- a/certbot-dns-google/certbot_dns_google/__init__.py +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -51,8 +51,16 @@ are automatically obtained by certbot through the `metadata service :caption: Example credentials file: { - "type": "service_account", - ... + "type": "service_account", + "project_id": "...", + "private_key_id": "...", + "private_key": "...", + "client_email": "...", + "client_id": "...", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "..." } The path to this file can be provided interactively or using the diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index 363c5e079..3a2686a63 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -32,13 +32,9 @@ class Authenticator(dns_common.DNSAuthenticator): 'for DNS).') ttl = 60 - def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None - @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) + super().add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help=('Path to Google Cloud DNS service account JSON file. (See {0} for' + 'information about creating a service account and {1} for information about the' + diff --git a/certbot-dns-google/docs/_ext/jsonlexer.py b/certbot-dns-google/docs/_ext/jsonlexer.py deleted file mode 100644 index 1ad004d2b..000000000 --- a/certbot-dns-google/docs/_ext/jsonlexer.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Copied from https://stackoverflow.com/a/16863232""" - -def setup(app): - # enable Pygments json lexer - try: - import pygments - if pygments.__version__ >= '1.5': - # use JSON lexer included in recent versions of Pygments - from pygments.lexers import JsonLexer - else: - # use JSON lexer from pygments-json if installed - from pygson.json_lexer import JSONLexer as JsonLexer - except ImportError: - pass # not fatal if we have old (or no) Pygments and no pygments-json - else: - app.add_lexer('json', JsonLexer()) diff --git a/certbot-dns-google/docs/conf.py b/certbot-dns-google/docs/conf.py index 06bb99f46..f4c1f661e 100644 --- a/certbot-dns-google/docs/conf.py +++ b/certbot-dns-google/docs/conf.py @@ -35,8 +35,7 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', - 'jsonlexer'] + 'sphinx.ext.viewcode'] autodoc_member_order = 'bysource' autodoc_default_flags = ['show-inheritance'] diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index e8b26eceb..dd43f4992 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -39,7 +39,7 @@ setup( description="Google Cloud DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 7de5f1d67..aa3ff35b5 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -26,14 +26,14 @@ PROJECT_ID = "test-test-1" class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_google._internal.dns_google import Authenticator path = os.path.join(self.tempdir, 'file.json') open(path, "wb").close() - super(AuthenticatorTest, self).setUp() + super().setUp() self.config = mock.MagicMock(google_credentials=path, google_propagation_seconds=0) # don't wait during tests diff --git a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py index c1b5e066f..b1649cf61 100644 --- a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py @@ -1,6 +1,7 @@ """DNS Authenticator for Linode.""" import logging import re +from typing import Optional from lexicon.providers import linode from lexicon.providers import linode4 @@ -10,6 +11,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): description = 'Obtain certificates using a DNS TXT record (if you are using Linode for DNS).' def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=120) + super().add_parser_arguments(add, default_propagation_seconds=120) add('credentials', help='Linode credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -56,6 +58,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_linode_client().del_txt_record(domain, validation_name, validation) def _get_linode_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") api_key = self.credentials.conf('key') api_version = self.credentials.conf('version') if api_version == '': @@ -81,7 +85,7 @@ class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, api_version): - super(_LinodeLexiconClient, self).__init__() + super().__init__() self.api_version = api_version diff --git a/certbot-dns-linode/local-oldest-requirements.txt b/certbot-dns-linode/local-oldest-requirements.txt index a8bd7449a..1307698d4 100644 --- a/certbot-dns-linode/local-oldest-requirements.txt +++ b/certbot-dns-linode/local-oldest-requirements.txt @@ -1,4 +1,3 @@ # Remember to update setup.py to match the package versions below. acme[dev]==0.31.0 certbot[dev]==1.1.0 -dns-lexicon==2.2.3 diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 63da31df2..293dc25c3 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,11 +4,11 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'dns-lexicon>=2.2.3', + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] @@ -35,7 +35,7 @@ setup( description="Linode DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-linode/tests/dns_linode_test.py b/certbot-dns-linode/tests/dns_linode_test.py index fb9b1aa93..9861d2ab0 100644 --- a/certbot-dns-linode/tests/dns_linode_test.py +++ b/certbot-dns-linode/tests/dns_linode_test.py @@ -22,7 +22,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write({"linode_key": TOKEN}, path) diff --git a/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py b/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py index d5b499c72..ed90b63d9 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py +++ b/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py @@ -1,5 +1,6 @@ """DNS Authenticator for LuaDNS DNS.""" import logging +from typing import Optional from lexicon.providers import luadns import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='LuaDNS credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -55,6 +57,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_luadns_client().del_txt_record(domain, validation_name, validation) def _get_luadns_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _LuaDNSLexiconClient(self.credentials.conf('email'), self.credentials.conf('token'), self.ttl) @@ -66,7 +70,7 @@ class _LuaDNSLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, email, token, ttl): - super(_LuaDNSLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('luadns', { 'ttl': ttl, diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 61b9dfa78..9ecacdd26 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,12 +4,12 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] @@ -36,7 +36,7 @@ setup( description="LuaDNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-luadns/tests/dns_luadns_test.py b/certbot-dns-luadns/tests/dns_luadns_test.py index a1242582f..7592e2323 100644 --- a/certbot-dns-luadns/tests/dns_luadns_test.py +++ b/certbot-dns-luadns/tests/dns_luadns_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_luadns._internal.dns_luadns import Authenticator diff --git a/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py b/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py index d328d80ce..ce46ad835 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py +++ b/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py @@ -1,5 +1,6 @@ """DNS Authenticator for NS1 DNS.""" import logging +from typing import Optional from lexicon.providers import nsone import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='NS1 credentials file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -54,6 +56,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_nsone_client().del_txt_record(domain, validation_name, validation) def _get_nsone_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _NS1LexiconClient(self.credentials.conf('api-key'), self.ttl) @@ -63,7 +67,7 @@ class _NS1LexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, ttl): - super(_NS1LexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('nsone', { 'ttl': ttl, diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index d60412f27..cf88a118a 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,12 +4,12 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] @@ -36,7 +36,7 @@ setup( description="NS1 DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-nsone/tests/dns_nsone_test.py b/certbot-dns-nsone/tests/dns_nsone_test.py index 83371252f..3754f9811 100644 --- a/certbot-dns-nsone/tests/dns_nsone_test.py +++ b/certbot-dns-nsone/tests/dns_nsone_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_nsone._internal.dns_nsone import Authenticator diff --git a/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py index 11ae6b8f0..54fd6d791 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py +++ b/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py @@ -1,5 +1,6 @@ """DNS Authenticator for OVH DNS.""" import logging +from typing import Optional from lexicon.providers import ovh import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='OVH credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -60,6 +62,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_ovh_client().del_txt_record(domain, validation_name, validation) def _get_ovh_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _OVHLexiconClient( self.credentials.conf('endpoint'), self.credentials.conf('application-key'), @@ -75,7 +79,7 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, endpoint, application_key, application_secret, consumer_key, ttl): - super(_OVHLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('ovh', { 'ttl': ttl, @@ -102,4 +106,4 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): if domain_name in str(e) and str(e).endswith('not found'): return - super(_OVHLexiconClient, self)._handle_general_error(e, domain_name) + super()._handle_general_error(e, domain_name) diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt index c55e0d570..1307698d4 100644 --- a/certbot-dns-ovh/local-oldest-requirements.txt +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -1,4 +1,3 @@ # Remember to update setup.py to match the package versions below. acme[dev]==0.31.0 certbot[dev]==1.1.0 -dns-lexicon==2.7.14 diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 4d6131ff7..e4dd4d712 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,12 +4,12 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] @@ -36,7 +36,7 @@ setup( description="OVH DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-ovh/tests/dns_ovh_test.py b/certbot-dns-ovh/tests/dns_ovh_test.py index dd0f3058b..7f93967eb 100644 --- a/certbot-dns-ovh/tests/dns_ovh_test.py +++ b/certbot-dns-ovh/tests/dns_ovh_test.py @@ -23,7 +23,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_ovh._internal.dns_ovh import Authenticator diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py index adebdd963..28fd27eec 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py @@ -1,5 +1,6 @@ """DNS Authenticator using RFC 2136 Dynamic Updates.""" import logging +from typing import Optional import dns.flags import dns.message @@ -15,6 +16,7 @@ import zope.interface from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -43,12 +45,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 120 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) + super().add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help='RFC 2136 credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -80,6 +82,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_rfc2136_client().del_txt_record(validation_name, validation) def _get_rfc2136_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _RFC2136Client(self.credentials.conf('server'), int(self.credentials.conf('port') or self.PORT), self.credentials.conf('name'), diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 18d006732..a19753e79 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="RFC 2136 DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py index dc4a73a04..c2b80defe 100644 --- a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py @@ -28,7 +28,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def setUp(self): from certbot_dns_rfc2136._internal.dns_rfc2136 import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write(VALID_CONFIG, path) diff --git a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py index 4dda13f1d..9e470bdde 100644 --- a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py @@ -37,7 +37,7 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 10 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.r53 = boto3.client("route53") self._resource_records: DefaultDict[str, List[Dict[str, str]]] = collections.defaultdict(list) diff --git a/certbot-dns-route53/certbot_dns_route53/authenticator.py b/certbot-dns-route53/certbot_dns_route53/authenticator.py index 2987934a1..060d2fa38 100644 --- a/certbot-dns-route53/certbot_dns_route53/authenticator.py +++ b/certbot-dns-route53/certbot_dns_route53/authenticator.py @@ -18,4 +18,4 @@ class Authenticator(dns_route53.Authenticator): def __init__(self, *args, **kwargs): warnings.warn("The 'authenticator' module was renamed 'dns_route53'", DeprecationWarning) - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index fdc931bbe..b1c1d786c 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -36,7 +36,7 @@ setup( description="Route53 DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-route53/tests/dns_route53_test.py b/certbot-dns-route53/tests/dns_route53_test.py index 1fd191c69..69b6b115d 100644 --- a/certbot-dns-route53/tests/dns_route53_test.py +++ b/certbot-dns-route53/tests/dns_route53_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(unittest.TestCase, dns_test_common.BaseAuthenticatorTest def setUp(self): from certbot_dns_route53._internal.dns_route53 import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() self.config = mock.MagicMock() diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py index 67cfb2e97..668c5e1f8 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py @@ -1,12 +1,15 @@ """DNS Authenticator for Sakura Cloud DNS.""" import logging +from typing import Optional from lexicon.providers import sakuracloud import zope.interface +from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -26,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + super().__init__(*args, **kwargs) + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments( + super().add_parser_arguments( add, default_propagation_seconds=90) add('credentials', help='Sakura Cloud credentials file.') @@ -60,6 +63,8 @@ class Authenticator(dns_common.DNSAuthenticator): domain, validation_name, validation) def _get_sakuracloud_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _SakuraCloudLexiconClient( self.credentials.conf('api-token'), self.credentials.conf('api-secret'), @@ -73,7 +78,7 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_token, api_secret, ttl): - super(_SakuraCloudLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('sakuracloud', { 'ttl': ttl, @@ -87,4 +92,4 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return None # Expected errors when zone name guess is wrong - return super(_SakuraCloudLexiconClient, self)._handle_http_error(e, domain_name) + return super()._handle_http_error(e, domain_name) diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 55f140d76..353111ec2 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,11 +4,11 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'dns-lexicon>=2.1.23', + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] @@ -35,7 +35,7 @@ setup( description="Sakura Cloud DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py index af94336b3..1c64df372 100644 --- a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py +++ b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_sakuracloud._internal.dns_sakuracloud import Authenticator diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index aab489315..9fb81efaa 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -102,7 +102,7 @@ class NginxConfigurator(common.Installer): """ version = kwargs.pop("version", None) openssl_version = kwargs.pop("openssl_version", None) - super(NginxConfigurator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) # Files to save self.save_notes = "" @@ -119,7 +119,6 @@ class NginxConfigurator(common.Installer): self._chall_out = 0 # These will be set in the prepare function - self.parser: Optional[parser.NginxParser] = None self.version = version self.openssl_version = openssl_version self._enhance_func = {"redirect": self._enable_redirect, @@ -127,6 +126,7 @@ class NginxConfigurator(common.Installer): "staple-ocsp": self._enable_ocsp_stapling} self.reverter.recovery_routine() + self.parser: parser.NginxParser @property def mod_ssl_conf_src(self): @@ -1110,7 +1110,7 @@ class NginxConfigurator(common.Installer): :raises .errors.PluginError: If unable to recover the configuration """ - super(NginxConfigurator, self).recovery_routine() + super().recovery_routine() self.new_vhost = None self.parser.load() @@ -1133,7 +1133,7 @@ class NginxConfigurator(common.Installer): the function is unable to correctly revert the configuration """ - super(NginxConfigurator, self).rollback_checkpoints(rollback) + super().rollback_checkpoints(rollback) self.new_vhost = None self.parser.load() diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index abcdfbd53..716af0898 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -37,7 +37,7 @@ class NginxHttp01(common.ChallengePerformer): """ def __init__(self, configurator): - super(NginxHttp01, self).__init__(configurator) + super().__init__(configurator) self.challenge_conf = os.path.join( configurator.config.config_dir, "le_http_01_cert_challenge.conf") diff --git a/certbot-nginx/certbot_nginx/_internal/obj.py b/certbot-nginx/certbot_nginx/_internal/obj.py index 1511cba6d..44be0e598 100644 --- a/certbot-nginx/certbot_nginx/_internal/obj.py +++ b/certbot-nginx/certbot_nginx/_internal/obj.py @@ -1,7 +1,6 @@ """Module contains classes used by the Nginx Configurator.""" import re - from certbot.plugins import common ADD_HEADER_DIRECTIVE = 'add_header' @@ -36,7 +35,7 @@ class Addr(common.Addr): CANONICAL_UNSPECIFIED_ADDRESS = UNSPECIFIED_IPV4_ADDRESSES[0] def __init__(self, host, port, ssl, default, ipv6, ipv6only): - super(Addr, self).__init__((host, port)) + super().__init__((host, port)) self.ssl = ssl self.default = default self.ipv6 = ipv6 @@ -121,7 +120,7 @@ class Addr(common.Addr): def __hash__(self): # pylint: disable=useless-super-delegation # Python 3 requires explicit overridden for __hash__ # See certbot-apache/certbot_apache/_internal/obj.py for more information - return super(Addr, self).__hash__() + return super().__hash__() def super_eq(self, other): """Check ip/port equality, with IPv6 support. @@ -133,7 +132,7 @@ class Addr(common.Addr): self.tup[1]), self.ipv6) == \ common.Addr((other.CANONICAL_UNSPECIFIED_ADDRESS, other.tup[1]), other.ipv6) - return super(Addr, self).__eq__(other) + return super().__eq__(other) def __eq__(self, other): if isinstance(other, self.__class__): diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 8b353c095..28833b1f7 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -18,6 +18,7 @@ from certbot import errors from certbot.compat import os from certbot_nginx._internal import nginxparser from certbot_nginx._internal import obj +from certbot_nginx._internal.nginxparser import UnspacedList logger = logging.getLogger(__name__) @@ -243,6 +244,8 @@ class NginxParser: tree = self.parsed[filename] if ext: filename = filename + os.path.extsep + ext + if not isinstance(tree, UnspacedList): + raise ValueError(f"Error tree {tree} is not an UnspacedList") try: if lazy and not tree.is_dirty(): continue diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 37392ea80..d4d332c47 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -1,3 +1,5 @@ +# type: ignore +# This module is not used for now, so we just skip type check for the sake of simplicity. """ This file contains parsing routines and object classes to help derive meaning from raw lists of tokens from pyparsing. """ @@ -120,7 +122,7 @@ class Statements(Parsable): precede any more statements. """ def __init__(self, parent=None): - super(Statements, self).__init__(parent) + super().__init__(parent) self._trailing_whitespace = None # ======== Begin overridden functions @@ -165,7 +167,7 @@ class Statements(Parsable): def dump(self, include_spaces=False): """ Dumps this object by first dumping each statement, then appending its trailing whitespace (if `include_spaces` is set) """ - data = super(Statements, self).dump(include_spaces) + data = super().dump(include_spaces) if include_spaces and self._trailing_whitespace is not None: return data + [self._trailing_whitespace] return data @@ -204,7 +206,7 @@ class Sentence(Parsable): :returns: whether this lists is parseable by `Sentence`. """ return isinstance(lists, list) and len(lists) > 0 and \ - all(isinstance(elem, str) for elem in lists) + all(isinstance(elem, str) for elem in lists) def parse(self, raw_list, add_spaces=False): """ Parses a list of string types into this object. @@ -212,7 +214,7 @@ class Sentence(Parsable): if add_spaces: raw_list = _space_list(raw_list) if not isinstance(raw_list, list) or \ - any(not isinstance(elem, str) for elem in raw_list): + any(not isinstance(elem, str) for elem in raw_list): raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") self._data = raw_list @@ -269,7 +271,7 @@ class Block(Parsable): contents = [["\n ", "content", " ", "1"], ["\n ", "content", " ", "2"], "\n"] """ def __init__(self, parent=None): - super(Block, self).__init__(parent) + super().__init__(parent) self.names: Sentence = None self.contents: Block = None @@ -283,7 +285,7 @@ class Block(Parsable): :returns: whether this lists is parseable by `Block`. """ return isinstance(lists, list) and len(lists) == 2 and \ - Sentence.should_parse(lists[0]) and isinstance(lists[1], list) + Sentence.should_parse(lists[0]) and isinstance(lists[1], list) def set_tabs(self, tabs=" "): """ Sets tabs by setting equivalent tabbing on names, then adding tabbing diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 8b638f28e..f42a6e85d 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -20,7 +20,7 @@ setup( description="Nginx plugin for Certbot", url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 9ccc3fc9e..e6ceca344 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -26,7 +26,7 @@ class NginxConfiguratorTest(util.NginxTest): def setUp(self): - super(NginxConfiguratorTest, self).setUp() + super().setUp() self.config = self.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) @@ -954,7 +954,7 @@ class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" def setUp(self): - super(InstallSslOptionsConfTest, self).setUp() + super().setUp() self.config = self.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) diff --git a/certbot-nginx/tests/display_ops_test.py b/certbot-nginx/tests/display_ops_test.py index 377255441..bcd455410 100644 --- a/certbot-nginx/tests/display_ops_test.py +++ b/certbot-nginx/tests/display_ops_test.py @@ -12,7 +12,7 @@ class SelectVhostMultiTest(util.NginxTest): """Tests for certbot_nginx._internal.display_ops.select_vhost_multiple.""" def setUp(self): - super(SelectVhostMultiTest, self).setUp() + super().setUp() nparser = parser.NginxParser(self.config_path) self.vhosts = nparser.get_vhosts() diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index d0e84fc83..f10e44859 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -51,7 +51,7 @@ class HttpPerformTest(util.NginxTest): ] def setUp(self): - super(HttpPerformTest, self).setUp() + super().setUp() config = self.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 35173d094..b062c4196 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -3,6 +3,7 @@ import glob import re import shutil import unittest +from typing import List from certbot import errors from certbot.compat import os diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index ee53e8e1e..383a15753 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -22,7 +22,7 @@ from certbot_nginx._internal import nginxparser class NginxTest(test_util.ConfigTestCase): def setUp(self): - super(NginxTest, self).setUp() + super().setUp() self.configuration = self.config self.config = None diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 9b37e9a1a..0220f1039 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,42 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.14.0 - master +## 1.16.0 - master + +### Added + +* + +### Changed + +* DNS plugins based on lexicon now require dns-lexicon >= v3.1.0 +* Use UTF-8 encoding for renewal configuration files +* Windows installer now cleans up old Certbot dependency packages + before installing the new ones to avoid version conflicts. + +### Fixed + +* Fix TypeError due to incompatibility with lexicon >= v3.6.0 + +More details about these changes can be found on our GitHub repo. + +## 1.15.0 - 2021-05-04 + +### Added + +* + +### Changed + +* + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + +## 1.14.0 - 2021-04-06 ### Added diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 790443bcc..4557ee399 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.14.0.dev0' +__version__ = '1.16.0.dev0' diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index d48fd419a..7d53ad649 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -14,9 +14,6 @@ from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW from certbot._internal.cli.cli_constants import DEPRECATED_OPTIONS from certbot._internal.cli.cli_constants import EXIT_ACTIONS from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE -from certbot._internal.cli.cli_constants import LEAUTO -from certbot._internal.cli.cli_constants import new_path_prefix -from certbot._internal.cli.cli_constants import old_path_fragment from certbot._internal.cli.cli_constants import SHORT_USAGE from certbot._internal.cli.cli_constants import VAR_MODIFIERS from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS @@ -47,7 +44,6 @@ from certbot._internal.plugins import disco as plugins_disco import certbot._internal.plugins.selection as plugin_selection import certbot.plugins.enhancements as enhancements - logger = logging.getLogger(__name__) @@ -247,8 +243,7 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): " to --server " + constants.STAGING_URI) helpful.add( "testing", "--debug", action="store_true", default=flag_default("debug"), - help="Show tracebacks in case of errors, and allow certbot-auto " - "execution on experimental platforms") + help="Show tracebacks in case of errors") helpful.add( [None, "certonly", "run"], "--debug-challenges", action="store_true", default=flag_default("debug_challenges"), diff --git a/certbot/certbot/_internal/cli/cli_constants.py b/certbot/certbot/_internal/cli/cli_constants.py index bd25f9bee..df815f2e6 100644 --- a/certbot/certbot/_internal/cli/cli_constants.py +++ b/certbot/certbot/_internal/cli/cli_constants.py @@ -1,29 +1,5 @@ """Certbot command line constants""" -import sys - -from certbot.compat import os - -# For help strings, figure out how the user ran us. -# When invoked from letsencrypt-auto, sys.argv[0] is something like: -# "/home/user/.local/share/certbot/bin/certbot" -# Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before -# running letsencrypt-auto (and sudo stops us from seeing if they did), so it -# should only be used for purposes where inability to detect letsencrypt-auto -# fails safely - -LEAUTO = "letsencrypt-auto" -if "CERTBOT_AUTO" in os.environ: - # if we're here, this is probably going to be certbot-auto, unless the - # user saved the script under a different name - LEAUTO = os.path.basename(os.environ["CERTBOT_AUTO"]) - -old_path_fragment = os.path.join(".local", "share", "letsencrypt") -new_path_prefix = os.path.abspath(os.path.join(os.sep, "opt", - "eff.org", "certbot", "venv")) -if old_path_fragment in sys.argv[0] or sys.argv[0].startswith(new_path_prefix): - cli_command = LEAUTO -else: - cli_command = "certbot" +cli_command = "certbot" # Argparse's help formatting has a lot of unhelpful peculiarities, so we want diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index 1f30261c1..8060b5e21 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -140,7 +140,7 @@ class CaseInsensitiveList(list): through the `helpful` wrapper. It is necessary due to special handling of command line arguments by `set_by_cli` in which the `type_func` is not applied.""" def __contains__(self, element): - return super(CaseInsensitiveList, self).__contains__(element.lower()) + return super().__contains__(element.lower()) def _user_agent_comment_type(value): diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index f185bdc26..73872584c 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -116,6 +116,8 @@ class HelpfulArgumentParser: # This is the only way to turn off overly verbose config flag documentation self.parser._add_config_file_help = False + self.verb: str + # Help that are synonyms for --help subcommands COMMANDS_TOPICS = ["command", "commands", "subcommand", "subcommands", "verbs"] diff --git a/certbot/certbot/_internal/cli/subparsers.py b/certbot/certbot/_internal/cli/subparsers.py index 822381d21..dabb26661 100644 --- a/certbot/certbot/_internal/cli/subparsers.py +++ b/certbot/certbot/_internal/cli/subparsers.py @@ -32,8 +32,7 @@ def _create_subparsers(helpful): " Currently --csr only works with the 'certonly' subcommand.") helpful.add("revoke", "--reason", dest="reason", - choices=CaseInsensitiveList(sorted(constants.REVOCATION_REASONS, - key=constants.REVOCATION_REASONS.get)), + choices=CaseInsensitiveList(constants.REVOCATION_REASONS.keys()), action=_EncodeReasonAction, default=flag_default("reason"), help="Specify reason for revoking certificate. (default: unspecified)") helpful.add("revoke", diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index ae24276ae..8c8a34220 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -2,7 +2,6 @@ import datetime import logging import platform -from typing import List from typing import Optional from cryptography.hazmat.backends import default_backend @@ -59,7 +58,7 @@ def determine_user_agent(config): ua = ("CertbotACMEClient/{0} ({1}; {2}{8}) Authenticator/{3} Installer/{4} " "({5}; flags: {6}) Py/{7}") if os.environ.get("CERTBOT_DOCS") == "1": - cli_command = "certbot(-auto)" + cli_command = "certbot" os_info = "OS_NAME OS_VERSION" python_version = "major.minor.patchlevel" else: @@ -255,6 +254,7 @@ class Client: acme = acme_from_config_key(config, self.account.key, self.account.regr) self.acme = acme + self.auth_handler: Optional[auth_handler.AuthHandler] if auth is not None: self.auth_handler = auth_handler.AuthHandler( auth, self.acme, self.account, self.config.pref_challs) @@ -391,7 +391,7 @@ class Client: return cert, chain, key, csr def _get_order_and_authorizations(self, csr_pem: str, - best_effort: bool) -> List[messages.OrderResource]: + best_effort: bool) -> messages.OrderResource: """Request a new order and complete its authorizations. :param str csr_pem: A CSR in PEM format. @@ -408,6 +408,9 @@ class Client: raise errors.Error("The currently selected ACME CA endpoint does" " not support issuing wildcard certificates.") + if not self.auth_handler: + raise errors.Error("No authorization handler has been set.") + # For a dry run, ensure we have an order with fresh authorizations if orderr and self.config.dry_run: deactivated, failed = self.auth_handler.deactivate_valid_authorizations(orderr) diff --git a/certbot/certbot/_internal/display/completer.py b/certbot/certbot/_internal/display/completer.py index a6c984195..b43859b19 100644 --- a/certbot/certbot/_internal/display/completer.py +++ b/certbot/certbot/_internal/display/completer.py @@ -1,5 +1,8 @@ """Provides Tab completion when prompting users for a path.""" import glob +from typing import Callable +from typing import Iterator +from typing import Optional # readline module is not available on all systems try: @@ -26,7 +29,9 @@ class Completer: """ def __init__(self): - self._iter = self._original_completer = self._original_delims = None + self._iter: Iterator[str] + self._original_completer: Optional[Callable] + self._original_delims: str def complete(self, text, state): """Provides path completion for use with readline. diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index d00302598..80b79d993 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -205,10 +205,9 @@ class _WindowsLockMechanism(_BaseLockMechanism): # Under Windows, filesystem.open will raise directly an EACCES error # if the lock file is already locked. fd = filesystem.open(self._path, open_mode, 0o600) - # The need for this "type: ignore" was fixed in - # https://github.com/python/typeshed/pull/3607 and included in - # newer versions of mypy so it can be removed when mypy is - # upgraded. + # This "type: ignore" is currently needed because msvcrt methods + # are only defined on Windows. See + # https://github.com/python/typeshed/blob/16ae4c61201cd8b96b8b22cdfb2ab9e89ba5bcf2/stdlib/msvcrt.pyi. msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) # type: ignore except (IOError, OSError) as err: if fd: @@ -224,10 +223,11 @@ class _WindowsLockMechanism(_BaseLockMechanism): def release(self): """Release the lock.""" try: - # The need for this "type: ignore" was fixed in - # https://github.com/python/typeshed/pull/3607 and included in - # newer versions of mypy so it can be removed when mypy is - # upgraded. + if not self._fd: + raise errors.Error("The lock has not been acquired first.") + # This "type: ignore" is currently needed because msvcrt methods + # are only defined on Windows. See + # https://github.com/python/typeshed/blob/16ae4c61201cd8b96b8b22cdfb2ab9e89ba5bcf2/stdlib/msvcrt.pyi. msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) # type: ignore os.close(self._fd) diff --git a/certbot/certbot/_internal/log.py b/certbot/certbot/_internal/log.py index d8a55e27d..7338578a7 100644 --- a/certbot/certbot/_internal/log.py +++ b/certbot/certbot/_internal/log.py @@ -109,11 +109,12 @@ def post_arg_parse_setup(config): root_logger.addHandler(file_handler) root_logger.removeHandler(memory_handler) - temp_handler = memory_handler.target # pylint: disable=no-member + temp_handler = getattr(memory_handler, 'target', None) memory_handler.setTarget(file_handler) # pylint: disable=no-member memory_handler.flush(force=True) # pylint: disable=unexpected-keyword-arg memory_handler.close() - temp_handler.close() + if temp_handler: + temp_handler.close() if config.quiet: level = constants.QUIET_LOGGING_LEVEL @@ -170,7 +171,7 @@ class ColoredStreamHandler(logging.StreamHandler): """ def __init__(self, stream=None): - super(ColoredStreamHandler, self).__init__(stream) + super().__init__(stream) self.colored = (sys.stderr.isatty() if stream is None else stream.isatty()) self.red_level = logging.WARNING @@ -184,7 +185,7 @@ class ColoredStreamHandler(logging.StreamHandler): :rtype: str """ - out = super(ColoredStreamHandler, self).format(record) + out = super().format(record) if self.colored and record.levelno >= self.red_level: return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) return out @@ -199,14 +200,14 @@ class MemoryHandler(logging.handlers.MemoryHandler): """ def __init__(self, target=None, capacity=10000): # capacity doesn't matter because should_flush() is overridden - super(MemoryHandler, self).__init__(capacity, target=target) + super().__init__(capacity, target=target) def close(self): """Close the memory handler, but don't set the target to None.""" # This allows the logging module which may only have a weak # reference to the target handler to properly flush and close it. - target = self.target - super(MemoryHandler, self).close() + target = getattr(self, 'target') + super().close() self.target = target def flush(self, force=False): # pylint: disable=arguments-differ @@ -220,7 +221,7 @@ class MemoryHandler(logging.handlers.MemoryHandler): # This method allows flush() calls in logging.shutdown to be a # noop so we can control when this handler is flushed. if force: - super(MemoryHandler, self).flush() + super().flush() def shouldFlush(self, record): """Should the buffer be automatically flushed? @@ -248,7 +249,7 @@ class TempHandler(logging.StreamHandler): self._workdir = tempfile.mkdtemp() self.path = os.path.join(self._workdir, 'log') stream = util.safe_open(self.path, mode='w', chmod=0o600) - super(TempHandler, self).__init__(stream) + super().__init__(stream) self._delete = True def emit(self, record): @@ -258,7 +259,7 @@ class TempHandler(logging.StreamHandler): """ self._delete = False - super(TempHandler, self).emit(record) + super().emit(record) def close(self): """Close the handler and the temporary log file. @@ -274,7 +275,7 @@ class TempHandler(logging.StreamHandler): if self._delete: shutil.rmtree(self._workdir) self._delete = False - super(TempHandler, self).close() + super().close() finally: self.release() diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 4483599db..89ecfad3b 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -4,6 +4,9 @@ import functools import logging.handlers import sys +from contextlib import contextmanager +from typing import Generator +from typing import IO from typing import Iterable from typing import List from typing import Optional @@ -643,7 +646,9 @@ def _delete_if_appropriate(config): # don't delete if the archive_dir is used by some other lineage archive_dir = storage.full_archive_path( - configobj.ConfigObj(storage.renewal_file_for_certname(config, config.certname)), + configobj.ConfigObj( + storage.renewal_file_for_certname(config, config.certname), + encoding='utf-8', default_encoding='utf-8'), config, config.certname) try: cert_manager.match_and_check_overlaps(config, [lambda x: archive_dir], @@ -1271,12 +1276,8 @@ def renew_cert(config, plugins, lineage): :raises errors.PluginSelectionError: MissingCommandlineFlag if supplied parameters do not pass """ - try: - # installers are used in auth mode to determine domain names - installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly") - except errors.PluginSelectionError as e: - logger.info("Could not choose appropriate plugin: %s", e) - raise + # installers are used in auth mode to determine domain names + installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly") le_client = _init_le_client(config, auth, installer) renewed_lineage = _get_and_save_cert(le_client, config, lineage=lineage) @@ -1314,12 +1315,8 @@ def certonly(config, plugins): """ # SETUP: Select plugins and construct a client instance - try: - # installers are used in auth mode to determine domain names - installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly") - except errors.PluginSelectionError as e: - logger.info("Could not choose appropriate plugin: %s", e) - raise + # installers are used in auth mode to determine domain names + installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly") le_client = _init_le_client(config, auth, installer) @@ -1387,26 +1384,36 @@ def make_or_verify_needed_dirs(config): util.make_or_verify_dir(hook_dir, strict=config.strict_permissions) -def set_displayer(config): - """Set the displayer +@contextmanager +def make_displayer(config: configuration.NamespaceConfig + ) -> Generator[Union[display_util.NoninteractiveDisplay, + display_util.FileDisplay], None, None]: + """Creates a display object appropriate to the flags in the supplied config. :param config: Configuration object - :type config: interfaces.IConfig - :returns: `None` - :rtype: None + :returns: Display object implementing :class:`certbot.interfaces.IDisplay` """ + displayer: Union[None, display_util.NoninteractiveDisplay, + display_util.FileDisplay] = None + devnull: Optional[IO] = None + if config.quiet: config.noninteractive_mode = True - displayer: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] =\ - display_util.NoninteractiveDisplay(open(os.devnull, "w")) + devnull = open(os.devnull, "w") + displayer = display_util.NoninteractiveDisplay(devnull) elif config.noninteractive_mode: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: - displayer = display_util.FileDisplay(sys.stdout, - config.force_interactive) - zope.component.provideUtility(displayer) + displayer = display_util.FileDisplay( + sys.stdout, config.force_interactive) + + try: + yield displayer + finally: + if devnull: + devnull.close() def main(cli_args=None): @@ -1451,11 +1458,12 @@ def main(cli_args=None): if config.func != plugins_cmd: # pylint: disable=comparison-with-callable raise - set_displayer(config) - # Reporter report = reporter.Reporter(config) zope.component.provideUtility(report) util.atexit_register(report.print_messages) - return config.func(config, plugins) + with make_displayer(config) as displayer: + zope.component.provideUtility(displayer) + + return config.func(config, plugins) diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index 4097544e8..744062968 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -1,18 +1,20 @@ """Utilities for plugins discovery and selection.""" -from collections.abc import Mapping import itertools import logging import sys +from collections.abc import Mapping from typing import Dict +from typing import Optional +from typing import Union import pkg_resources import zope.interface import zope.interface.verify - from certbot import errors from certbot import interfaces from certbot._internal import constants from certbot.compat import os +from certbot.errors import Error logger = logging.getLogger(__name__) @@ -44,15 +46,15 @@ class PluginEntryPoint: # this object is mutable, don't allow it to be hashed! __hash__ = None # type: ignore - def __init__(self, entry_point, with_prefix=False): + def __init__(self, entry_point: pkg_resources.EntryPoint, with_prefix=False): self.name = self.entry_point_to_plugin_name(entry_point, with_prefix) - self.plugin_cls = entry_point.load() + self.plugin_cls: interfaces.IPluginFactory = entry_point.load() self.entry_point = entry_point - self.warning_message = None - self._initialized = None - self._prepared = None + self.warning_message: Optional[str] = None + self._initialized: Optional[interfaces.IPlugin] = None + self._prepared: Optional[Union[bool, Error]] = None self._hidden = False - self._long_description = None + self._long_description: Optional[str] = None def check_name(self, name): """Check if the name refers to this plugin.""" @@ -118,12 +120,15 @@ class PluginEntryPoint: """Memoized plugin initialization.""" if not self.initialized: self.entry_point.require() # fetch extras! - self._initialized = self.plugin_cls(config, self.name) + # TODO: remove type ignore once the interface becomes a proper + # abstract class (using abc) that mypy understands. + self._initialized = self.plugin_cls(config, self.name) # type: ignore return self._initialized def verify(self, ifaces): """Verify that the plugin conforms to the specified interfaces.""" - assert self.initialized + if not self.initialized: + raise ValueError("Plugin is not initialized.") for iface in ifaces: # zope.interface.providedBy(plugin) try: zope.interface.verify.verifyObject(iface, self.init()) @@ -144,10 +149,13 @@ class PluginEntryPoint: def prepare(self): """Memoized plugin preparation.""" - assert self.initialized + if self._initialized is None: + raise ValueError("Plugin is not initialized.") if self._prepared is None: try: - self._initialized.prepare() + # TODO: remove type ignore once the interface becomes a proper + # abstract class (using abc) that mypy understands. + self._initialized.prepare() # type: ignore except errors.MisconfigurationError as error: logger.debug("Misconfigured %r: %s", self, error, exc_info=True) self._prepared = error @@ -247,8 +255,10 @@ class PluginsRegistry(Mapping): plugin_ep = PluginEntryPoint(entry_point, with_prefix) if plugin_ep.name in plugins: other_ep = plugins[plugin_ep.name] + plugin1 = plugin_ep.entry_point.dist.key if plugin_ep.entry_point.dist else "unknown" + plugin2 = other_ep.entry_point.dist.key if other_ep.entry_point.dist else "unknown" raise Exception("Duplicate plugin name {0} from {1} and {2}.".format( - plugin_ep.name, plugin_ep.entry_point.dist.key, other_ep.entry_point.dist.key)) + plugin_ep.name, plugin1, plugin2)) if interfaces.IPluginFactory.providedBy(plugin_ep.plugin_cls): plugins[plugin_ep.name] = plugin_ep else: # pragma: no cover diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index ed2e0559e..dec73a1ed 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -43,13 +43,30 @@ class Authenticator(common.Plugin): '$CERTBOT_REMAINING_CHALLENGES will be equal to the number of challenges that ' 'remain after the current one, and $CERTBOT_ALL_DOMAINS contains a comma-separated ' 'list of all domains that are challenged for the current certificate.') + # Include the full stop at the end of the FQDN in the instructions below for the null + # label of the DNS root, as stated in section 3.1 of RFC 1035. While not necessary + # for most day to day usage of hostnames, when adding FQDNs to a DNS zone editor, this + # full stop is often mandatory. Without a full stop, the entered name is often seen as + # relative to the DNS zone origin, which could lead to entries for, e.g.: + # _acme-challenge.example.com.example.com. For users unaware of this subtle detail, + # including the trailing full stop in the DNS instructions below might avert this issue. _DNS_INSTRUCTIONS = """\ -Please deploy a DNS TXT record under the name -{domain} with the following value: +Please deploy a DNS TXT record under the name: + +{domain}. + +with the following value: {validation} - -Before continuing, verify the record is deployed.""" +""" + _DNS_VERIFY_INSTRUCTIONS = """ +Before continuing, verify the TXT record has been deployed. Depending on the DNS +provider, this may take some time, from a few seconds to multiple minutes. You can +check if it has finished deploying with aid of online tools, such as the Google +Admin Toolbox: https://toolbox.googleapps.com/apps/dig/#TXT/{domain}. +Look for one or more bolded line(s) below the line ';ANSWER'. It should show the +value(s) you've just added. +""" _HTTP_INSTRUCTIONS = """\ Create a file containing just this data: @@ -71,7 +88,7 @@ permitted by DNS standards.) """ def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() self.env: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] = {} @@ -114,11 +131,15 @@ permitted by DNS standards.) def perform(self, achalls): # pylint: disable=missing-function-docstring responses = [] - for achall in achalls: + last_dns_achall = 0 + for i, achall in enumerate(achalls): + if isinstance(achall.chall, challenges.DNS01): + last_dns_achall = i + for i, achall in enumerate(achalls): if self.conf('auth-hook'): self._perform_achall_with_script(achall, achalls) else: - self._perform_achall_manually(achall) + self._perform_achall_manually(achall, i == last_dns_achall) responses.append(achall.response(achall.account_key)) return responses @@ -136,7 +157,7 @@ permitted by DNS standards.) env['CERTBOT_AUTH_OUTPUT'] = out.strip() self.env[achall] = env - def _perform_achall_manually(self, achall): + def _perform_achall_manually(self, achall, last_dns_achall=False): validation = achall.validation(achall.account_key) if isinstance(achall.chall, challenges.HTTP01): msg = self._HTTP_INSTRUCTIONS.format( @@ -152,7 +173,15 @@ permitted by DNS standards.) if self.subsequent_dns_challenge: # 2nd or later dns-01 challenge msg += self._SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS + elif self.subsequent_any_challenge: + # 1st dns-01 challenge, but 2nd or later *any* challenge, so + # instruct user not to remove any previous http-01 challenge + msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS self.subsequent_dns_challenge = True + if last_dns_achall: + # last dns-01 challenge + msg += self._DNS_VERIFY_INSTRUCTIONS.format( + domain=achall.validation_domain_name(achall.domain)) elif self.subsequent_any_challenge: # 2nd or later challenge of another type msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index bf7e505b9..f0cce002f 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -134,19 +134,10 @@ def choose_plugin(prepared, question): opts = [plugin_ep.description_with_name + (" [Misconfigured]" if plugin_ep.misconfigured else "") for plugin_ep in prepared] - names = set(plugin_ep.name for plugin_ep in prepared) while True: disp = z_util(interfaces.IDisplay) - if "CERTBOT_AUTO" in os.environ and names == {"apache", "nginx"}: - # The possibility of being offered exactly apache and nginx here - # is new interactivity brought by https://github.com/certbot/certbot/issues/4079, - # so set apache as a default for those kinds of non-interactive use - # (the user will get a warning to set --non-interactive or --force-interactive) - apache_idx = [n for n, p in enumerate(prepared) if p.name == "apache"][0] - code, index = disp.menu(question, opts, default=apache_idx) - else: - code, index = disp.menu(question, opts, force_interactive=True) + code, index = disp.menu(question, opts, force_interactive=True) if code == display_util.OK: plugin_ep = prepared[index] diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index 833651346..5fb29671f 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -1,9 +1,8 @@ """Standalone Authenticator.""" import collections +import errno import logging import socket -# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi -from socket import errno as socket_errors # type: ignore from typing import DefaultDict from typing import Dict from typing import Set @@ -120,7 +119,7 @@ class Authenticator(common.Plugin): description = "Spin up a temporary webserver" def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.served: ServedType = collections.defaultdict(set) @@ -187,13 +186,13 @@ class Authenticator(common.Plugin): def _handle_perform_error(error): - if error.socket_error.errno == socket_errors.EACCES: + if error.socket_error.errno == errno.EACCES: raise errors.PluginError( "Could not bind TCP port {0} because you don't have " "the appropriate permissions (for example, you " "aren't running this program as " "root).".format(error.port)) - if error.socket_error.errno == socket_errors.EADDRINUSE: + if error.socket_error.errno == errno.EADDRINUSE: display = zope.component.getUtility(interfaces.IDisplay) msg = ( "Could not bind TCP port {0} because it is already in " diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index c5b436b65..3473d3c8e 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -66,7 +66,7 @@ to serve all files under specified web root ({0}).""" return [challenges.HTTP01] def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.full_roots: Dict[str, str] = {} self.performed: DefaultDict[str, Set[AnnotatedChallenge]] = collections.defaultdict(set) # stack of dirs successfully created by this authenticator @@ -250,7 +250,7 @@ class _WebrootPathAction(argparse.Action): """Action class for parsing webroot_path.""" def __init__(self, *args, **kwargs): - super(_WebrootPathAction, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._domain_before_webroot = False def __call__(self, parser, namespace, webroot_path, option_string=None): diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index 0268fbf34..295a2d4c5 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -29,7 +29,7 @@ class Reporter: LOW_PRIORITY = 2 """Low priority constant. See `add_message`.""" - _msg_type = collections.namedtuple('ReporterMsg', 'priority text on_crash') + _msg_type = collections.namedtuple('_msg_type', 'priority text on_crash') def __init__(self, config): self.messages: queue.PriorityQueue[Reporter._msg_type] = queue.PriorityQueue() diff --git a/certbot/certbot/_internal/snap_config.py b/certbot/certbot/_internal/snap_config.py index 1ba53cad4..f7832cd55 100644 --- a/certbot/certbot/_internal/snap_config.py +++ b/certbot/certbot/_internal/snap_config.py @@ -80,7 +80,7 @@ def prepare_env(cli_args: List[str]) -> List[str]: class _SnapdConnection(HTTPConnection): def __init__(self): - super(_SnapdConnection, self).__init__("localhost") + super().__init__("localhost") self.sock = None def connect(self): @@ -90,7 +90,7 @@ class _SnapdConnection(HTTPConnection): class _SnapdConnectionPool(HTTPConnectionPool): def __init__(self): - super(_SnapdConnectionPool, self).__init__("localhost") + super().__init__("localhost") def _new_conn(self): return _SnapdConnection() diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 218a5dd92..4551356d5 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -5,14 +5,14 @@ import logging import re import shutil import stat - from typing import Optional + import configobj -import parsedatetime -import pytz from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.hazmat.primitives.serialization import load_pem_private_key +import parsedatetime +import pytz import certbot from certbot import crypto_util @@ -67,13 +67,16 @@ def cert_path_for_cert_name(config: interfaces.IConfig, cert_name: str) -> str: """ cert_name_implied_conf = renewal_file_for_certname(config, cert_name) - return configobj.ConfigObj(cert_name_implied_conf)["fullchain"] + return configobj.ConfigObj( + cert_name_implied_conf, encoding='utf-8', default_encoding='utf-8')["fullchain"] def config_with_defaults(config=None): """Merge supplied config, if provided, on top of builtin defaults.""" - defaults_copy = configobj.ConfigObj(constants.RENEWER_DEFAULTS) - defaults_copy.merge(config if config is not None else configobj.ConfigObj()) + defaults_copy = configobj.ConfigObj( + constants.RENEWER_DEFAULTS, encoding='utf-8', default_encoding='utf-8') + defaults_copy.merge(config if config is not None else configobj.ConfigObj( + encoding='utf-8', default_encoding='utf-8')) return defaults_copy @@ -114,7 +117,7 @@ def write_renewal_config(o_filename, n_filename, archive_dir, target, relevant_d :rtype: configobj.ConfigObj """ - config = configobj.ConfigObj(o_filename) + config = configobj.ConfigObj(o_filename, encoding='utf-8', default_encoding='utf-8') config["version"] = certbot.__version__ config["archive_dir"] = archive_dir for kind in ALL_FOUR: @@ -196,7 +199,7 @@ def update_configuration(lineagename, archive_dir, target, cli_config): write_renewal_config(config_filename, temp_filename, archive_dir, target, values) filesystem.replace(temp_filename, config_filename) - return configobj.ConfigObj(config_filename) + return configobj.ConfigObj(config_filename, encoding='utf-8', default_encoding='utf-8') def get_link_target(link): @@ -324,7 +327,8 @@ def delete_files(config, certname): full_default_archive_dir = full_archive_path(None, config, certname) full_default_live_dir = _full_live_path(config, certname) try: - renewal_config = configobj.ConfigObj(renewal_filename) + renewal_config = configobj.ConfigObj( + renewal_filename, encoding='utf-8', default_encoding='utf-8') except configobj.ConfigObjError: # config is corrupted logger.warning("Could not parse %s. You may wish to manually " @@ -434,7 +438,8 @@ class RenewableCert(interfaces.RenewableCert): # systemwide renewal configuration; self.configfile should be # used to make and save changes. try: - self.configfile = configobj.ConfigObj(config_filename) + self.configfile = configobj.ConfigObj( + config_filename, encoding='utf-8', default_encoding='utf-8') except configobj.ConfigObjError: raise errors.CertStorageError( "error parsing {0}".format(config_filename)) diff --git a/certbot/certbot/achallenges.py b/certbot/certbot/achallenges.py index 7171c271c..a3ddbdcd3 100644 --- a/certbot/certbot/achallenges.py +++ b/certbot/certbot/achallenges.py @@ -18,15 +18,16 @@ Note, that all annotated challenges act as a proxy objects:: """ import logging +from typing import Type import josepy as jose from acme import challenges +from acme.challenges import Challenge logger = logging.getLogger(__name__) - class AnnotatedChallenge(jose.ImmutableMap): """Client annotated challenge. @@ -37,7 +38,7 @@ class AnnotatedChallenge(jose.ImmutableMap): """ __slots__ = ('challb',) - acme_type = NotImplemented + _acme_type: Type[Challenge] = NotImplemented def __getattr__(self, name): return getattr(self.challb, name) diff --git a/certbot/certbot/compat/_path.py b/certbot/certbot/compat/_path.py index 44c5e300f..a5db3a575 100644 --- a/certbot/certbot/compat/_path.py +++ b/certbot/certbot/compat/_path.py @@ -3,6 +3,10 @@ This compat module wraps os.path to forbid some functions. isort:skip_file """ + +# NB: Each function defined in compat._path is marked with "type: ignore" to avoid mypy +# to complain that a function is redefined (because we imported if first from os.path). + # pylint: disable=function-redefined from __future__ import absolute_import @@ -29,7 +33,7 @@ del ourselves, std_os_path, std_sys # Function os.path.realpath is broken on some versions of Python for Windows. -def realpath(*unused_args, **unused_kwargs): +def realpath(*unused_args, **unused_kwargs): # type: ignore """Method os.path.realpath() is forbidden""" raise RuntimeError('Usage of os.path.realpath() is forbidden. ' 'Use certbot.compat.filesystem.realpath() instead.') diff --git a/certbot/certbot/compat/os.py b/certbot/certbot/compat/os.py index 2f0899bd7..653877b50 100644 --- a/certbot/certbot/compat/os.py +++ b/certbot/certbot/compat/os.py @@ -6,12 +6,15 @@ This module is intended to replace standard os module throughout certbot project This module has the same API as the os module in the Python standard library except for the functions defined below. +isort:skip_file """ -# NOTE: If adding a new documented function to compat.os, ensure that it is added to the +# NB1: If adding a new documented function to compat.os, ensure that it is added to the # ':members:' list in certbot/docs/api/certbot.compat.os.rst. -# isort:skip_file +# NB2: Each function defined in compat.os is marked with "type: ignore" to avoid mypy +# to complain that a function is redefined (because we imported if first from os). + # pylint: disable=function-redefined from __future__ import absolute_import @@ -60,7 +63,7 @@ del ourselves, std_os, std_sys # Basically, it states that appropriate permissions will be set for the owner, nothing for the # group, appropriate permissions for the "Everyone" group, and all permissions to the # "Administrators" group + "System" user, as they can do everything anyway. -def chmod(*unused_args, **unused_kwargs): +def chmod(*unused_args, **unused_kwargs): # type: ignore """Method os.chmod() is forbidden""" raise RuntimeError('Usage of os.chmod() is forbidden. ' 'Use certbot.compat.filesystem.chmod() instead.') @@ -70,7 +73,7 @@ def chmod(*unused_args, **unused_kwargs): # this platform. In order to have a consistent behavior between Linux and Windows on Certbot files # and directories, the filesystem umask method must be used instead, since it implements umask for # Windows. -def umask(*unused_args, **unused_kwargs): +def umask(*unused_args, **unused_kwargs): # type: ignore """Method os.chmod() is forbidden""" raise RuntimeError('Usage of os.umask() is forbidden. ' 'Use certbot.compat.filesystem.umask() instead.') @@ -79,7 +82,7 @@ def umask(*unused_args, **unused_kwargs): # Because uid is not a concept on Windows, chown is useless. In fact, it is not even available # on Python for Windows. So to be consistent on both platforms for Certbot, this method is # always forbidden. -def chown(*unused_args, **unused_kwargs): +def chown(*unused_args, **unused_kwargs): # type: ignore """Method os.chown() is forbidden""" raise RuntimeError('Usage of os.chown() is forbidden.' 'Use certbot.compat.filesystem.copy_ownership_and_apply_mode() instead.') @@ -90,7 +93,7 @@ def chown(*unused_args, **unused_kwargs): # filesystem.open invokes the Windows native API `CreateFile` to ensure that permissions are # atomically set in case of file creation, or invokes filesystem.chmod to properly set the # permissions for the other cases. -def open(*unused_args, **unused_kwargs): +def open(*unused_args, **unused_kwargs): # type: ignore """Method os.open() is forbidden""" raise RuntimeError('Usage of os.open() is forbidden. ' 'Use certbot.compat.filesystem.open() instead.') @@ -98,7 +101,7 @@ def open(*unused_args, **unused_kwargs): # Very similarly to os.open, os.mkdir has the same effects on Windows and creates an unsecured # folder. So a similar mitigation to security.chmod is provided on this platform. -def mkdir(*unused_args, **unused_kwargs): +def mkdir(*unused_args, **unused_kwargs): # type: ignore """Method os.mkdir() is forbidden""" raise RuntimeError('Usage of os.mkdir() is forbidden. ' 'Use certbot.compat.filesystem.mkdir() instead.') @@ -109,7 +112,7 @@ def mkdir(*unused_args, **unused_kwargs): # that our modified os.mkdir is called on Windows, by monkey patching temporarily the mkdir method # on the original os module, executing the modified logic to correctly protect newly created # folders, then restoring original mkdir method in the os module. -def makedirs(*unused_args, **unused_kwargs): +def makedirs(*unused_args, **unused_kwargs): # type: ignore """Method os.makedirs() is forbidden""" raise RuntimeError('Usage of os.makedirs() is forbidden. ' 'Use certbot.compat.filesystem.makedirs() instead.') @@ -117,7 +120,7 @@ def makedirs(*unused_args, **unused_kwargs): # Because of the blocking strategy on file handlers on Windows, rename does not behave as expected # with POSIX systems: an exception will be raised if dst already exists. -def rename(*unused_args, **unused_kwargs): +def rename(*unused_args, **unused_kwargs): # type: ignore """Method os.rename() is forbidden""" raise RuntimeError('Usage of os.rename() is forbidden. ' 'Use certbot.compat.filesystem.replace() instead.') @@ -125,7 +128,7 @@ def rename(*unused_args, **unused_kwargs): # Behavior of os.replace is consistent between Windows and Linux. However, it is not supported on # Python 2.x. So, as for os.rename, we forbid it in favor of filesystem.replace. -def replace(*unused_args, **unused_kwargs): +def replace(*unused_args, **unused_kwargs): # type: ignore """Method os.replace() is forbidden""" raise RuntimeError('Usage of os.replace() is forbidden. ' 'Use certbot.compat.filesystem.replace() instead.') @@ -133,7 +136,7 @@ def replace(*unused_args, **unused_kwargs): # Results given by os.access are inconsistent or partial on Windows, because this platform is not # following the POSIX approach. -def access(*unused_args, **unused_kwargs): +def access(*unused_args, **unused_kwargs): # type: ignore """Method os.access() is forbidden""" raise RuntimeError('Usage of os.access() is forbidden. ' 'Use certbot.compat.filesystem.check_mode() or ' @@ -142,7 +145,7 @@ def access(*unused_args, **unused_kwargs): # On Windows os.stat call result is inconsistent, with a lot of flags that are not set or # meaningless. We need to use specialized functions from the certbot.compat.filesystem module. -def stat(*unused_args, **unused_kwargs): +def stat(*unused_args, **unused_kwargs): # type: ignore """Method os.stat() is forbidden""" raise RuntimeError('Usage of os.stat() is forbidden. ' 'Use certbot.compat.filesystem functions instead ' @@ -151,7 +154,7 @@ def stat(*unused_args, **unused_kwargs): # Method os.fstat has the same problem than os.stat, since it is the same function, # but accepting a file descriptor instead of a path. -def fstat(*unused_args, **unused_kwargs): +def fstat(*unused_args, **unused_kwargs): # type: ignore """Method os.stat() is forbidden""" raise RuntimeError('Usage of os.fstat() is forbidden. ' 'Use certbot.compat.filesystem functions instead ' @@ -163,7 +166,7 @@ def fstat(*unused_args, **unused_kwargs): # unconditionally, which allows to use more than 259 characters, and its string # representation is prepended with "\\?\". Problem is that it does it for any path, # and will make equality comparison fail with paths that will use the simple form. -def readlink(*unused_args, **unused_kwargs): +def readlink(*unused_args, **unused_kwargs): # type: ignore """Method os.readlink() is forbidden""" raise RuntimeError('Usage of os.readlink() is forbidden. ' 'Use certbot.compat.filesystem.realpath() instead.') diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 92ecd1679..4f1da9349 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -302,8 +302,7 @@ def verify_signed_payload(public_key, signature, payload, signature_hash_algorit with warnings.catch_warnings(): warnings.simplefilter("ignore") if isinstance(public_key, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = public_key.verifier( # type: ignore + verifier = public_key.verifier( signature, PKCS1v15(), signature_hash_algorithm ) verifier.update(payload) diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index 9443ae266..dc642586c 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -115,7 +115,7 @@ class FileDisplay: # see https://github.com/certbot/certbot/issues/3915 def __init__(self, outfile, force_interactive): - super(FileDisplay, self).__init__() + super().__init__() self.outfile = outfile self.force_interactive = force_interactive self.skipped_interaction = False @@ -481,7 +481,7 @@ class NoninteractiveDisplay: """An iDisplay implementation that never asks for interactive user input""" def __init__(self, outfile, *unused_args, **unused_kwargs): - super(NoninteractiveDisplay, self).__init__() + super().__init__() self.outfile = outfile def _interaction_fail(self, message, cli_flag, extra=""): diff --git a/certbot/certbot/errors.py b/certbot/certbot/errors.py index 48aebc267..cf0d3d283 100644 --- a/certbot/certbot/errors.py +++ b/certbot/certbot/errors.py @@ -53,7 +53,7 @@ class FailedChallenges(AuthorizationError): def __init__(self, failed_achalls): assert failed_achalls self.failed_achalls = failed_achalls - super(FailedChallenges, self).__init__() + super().__init__() def __str__(self): return "Failed authorization procedure. {0}".format( @@ -95,7 +95,7 @@ class StandaloneBindError(Error): """Standalone plugin bind error.""" def __init__(self, socket_error, port): - super(StandaloneBindError, self).__init__( + super().__init__( "Problem binding to port {0}: {1}".format(port, socket_error)) self.socket_error = socket_error self.port = port diff --git a/certbot/certbot/interfaces.py b/certbot/certbot/interfaces.py index ddbee8ddc..de9175def 100644 --- a/certbot/certbot/interfaces.py +++ b/certbot/certbot/interfaces.py @@ -1,5 +1,6 @@ """Certbot client interfaces.""" import abc +from typing import Optional import zope.interface @@ -327,7 +328,7 @@ class IInstaller(IPlugin): """ - def save(title=None, temporary=False): + def save(title: Optional[str] = None, temporary: bool = False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be @@ -349,7 +350,7 @@ class IInstaller(IPlugin): """ - def rollback_checkpoints(rollback=1): + def rollback_checkpoints(rollback: int = 1): """Revert `rollback` number of configuration checkpoints. :raises .PluginError: when configuration cannot be fully reverted diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index 06acefc69..4c33acbab 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -105,7 +105,7 @@ class Installer(Plugin): """ def __init__(self, *args, **kwargs): - super(Installer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.storage = PluginStorage(self.config, self.name) self.reverter = reverter.Reverter(self.config) diff --git a/certbot/certbot/plugins/dns_common.py b/certbot/certbot/plugins/dns_common.py index 5c0fbcba9..4459ac0fa 100644 --- a/certbot/certbot/plugins/dns_common.py +++ b/certbot/certbot/plugins/dns_common.py @@ -25,7 +25,7 @@ class DNSAuthenticator(common.Plugin): """Base class for DNS Authenticators""" def __init__(self, config, name): - super(DNSAuthenticator, self).__init__(config, name) + super().__init__(config, name) self._attempt_cleanup = False @@ -43,6 +43,9 @@ class DNSAuthenticator(common.Plugin): def prepare(self): # pylint: disable=missing-function-docstring pass + def more_info(self) -> str: # pylint: disable=missing-function-docstring + raise NotImplementedError() + def perform(self, achalls): # pylint: disable=missing-function-docstring self._setup_credentials() @@ -139,7 +142,8 @@ class DNSAuthenticator(common.Plugin): setattr(self.config, self.dest(key), os.path.abspath(os.path.expanduser(new_value))) - def _configure_credentials(self, key, label, required_variables=None, validator=None): + def _configure_credentials(self, key, label, required_variables=None, + validator=None) -> 'CredentialsConfiguration': """ As `_configure_file`, but for a credential configuration file. diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index 10efa5f85..32213999d 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -17,8 +17,10 @@ from certbot.plugins import dns_common # if Lexicon is not available, obviously. try: from lexicon.config import ConfigResolver + from lexicon.providers.base import Provider except ImportError: ConfigResolver = None # type: ignore + Provider = None # type: ignore logger = logging.getLogger(__name__) @@ -29,7 +31,7 @@ class LexiconClient: """ def __init__(self): - self.provider = None + self.provider: Provider def add_txt_record(self, domain, record_name, record_content): """ @@ -43,7 +45,7 @@ class LexiconClient: self._find_domain_id(domain) try: - self.provider.create_record(type='TXT', name=record_name, content=record_content) + self.provider.create_record(rtype='TXT', name=record_name, content=record_content) except RequestException as e: logger.debug('Encountered error adding TXT record: %s', e, exc_info=True) raise errors.PluginError('Error adding TXT record: {0}'.format(e)) @@ -65,7 +67,7 @@ class LexiconClient: return try: - self.provider.delete_record(type='TXT', name=record_name, content=record_content) + self.provider.delete_record(rtype='TXT', name=record_name, content=record_content) except RequestException as e: logger.debug('Encountered error deleting TXT record: %s', e, exc_info=True) diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index 1affcae4e..7a8df9329 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -1,22 +1,52 @@ """Base test class for DNS authenticators.""" +import typing import configobj import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore from acme import challenges from certbot import achallenges from certbot.compat import filesystem +from certbot.plugins.dns_common import DNSAuthenticator from certbot.tests import acme_util from certbot.tests import util as test_util +if typing.TYPE_CHECKING: + from typing_extensions import Protocol +else: + Protocol = object # type: ignore + + + +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore + + DOMAIN = 'example.com' KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) +class _AuthenticatorCallableTestCase(Protocol): + """Protocol describing a TestCase able to call a real DNSAuthenticator instance.""" + auth: DNSAuthenticator + + def assertTrue(self, *unused_args) -> None: + """ + See + https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertTrue + """ + ... + + def assertEqual(self, *unused_args) -> None: + """ + See + https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertEqual + """ + ... + + class BaseAuthenticatorTest: """ A base test class to reduce duplication between test code for DNS Authenticator Plugins. @@ -29,13 +59,13 @@ class BaseAuthenticatorTest: achall = achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.DNS01, domain=DOMAIN, account_key=KEY) - def test_more_info(self): + def test_more_info(self: _AuthenticatorCallableTestCase): self.assertTrue(isinstance(self.auth.more_info(), str)) # pylint: disable=no-member - def test_get_chall_pref(self): + def test_get_chall_pref(self: _AuthenticatorCallableTestCase): self.assertEqual(self.auth.get_chall_pref(None), [challenges.DNS01]) # pylint: disable=no-member - def test_parser_arguments(self): + def test_parser_arguments(self: _AuthenticatorCallableTestCase): m = mock.MagicMock() self.auth.add_parser_arguments(m) # pylint: disable=no-member diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index adeb1a268..203adf009 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -1,32 +1,79 @@ """Base test class for DNS authenticators built on Lexicon.""" +import typing +from unittest.mock import MagicMock import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException +from acme.challenges import Challenge from certbot import errors from certbot.plugins import dns_test_common +from certbot.plugins.dns_common_lexicon import LexiconClient +from certbot.plugins.dns_test_common import _AuthenticatorCallableTestCase from certbot.tests import util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore +if typing.TYPE_CHECKING: + from typing_extensions import Protocol +else: + Protocol = object # type: ignore + + + + DOMAIN = 'example.com' KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) + +class _AuthenticatorCallableLexiconTestCase(_AuthenticatorCallableTestCase, Protocol): + """ + Protocol describing a TestCase suitable to test challenges against + a mocked LexiconClient instance. + """ + mock_client: MagicMock + achall: Challenge + + +class _LexiconAwareTestCase(Protocol): + """ + Protocol describing a TestCase suitable to test a real LexiconClient instance. + """ + client: LexiconClient + provider_mock: MagicMock + + record_prefix: str + record_name: str + record_content: str + + DOMAIN_NOT_FOUND: Exception + GENERIC_ERROR: Exception + LOGIN_ERROR: Exception + UNKNOWN_LOGIN_ERROR: Exception + + def assertRaises(self, *unused_args) -> None: + """ + See + https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertRaises + """ + ... + + # These classes are intended to be subclassed/mixed in, so not all members are defined. # pylint: disable=no-member class BaseLexiconAuthenticatorTest(dns_test_common.BaseAuthenticatorTest): - def test_perform(self): + def test_perform(self: _AuthenticatorCallableLexiconTestCase): self.auth.perform([self.achall]) expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] self.assertEqual(expected, self.mock_client.mock_calls) - def test_cleanup(self): + def test_cleanup(self: _AuthenticatorCallableLexiconTestCase): self.auth._attempt_cleanup = True # _attempt_cleanup | pylint: disable=protected-access self.auth.cleanup([self.achall]) @@ -44,23 +91,23 @@ class BaseLexiconClientTest: record_name = record_prefix + "." + DOMAIN record_content = "bar" - def test_add_txt_record(self): + def test_add_txt_record(self: _LexiconAwareTestCase): self.client.add_txt_record(DOMAIN, self.record_name, self.record_content) - self.provider_mock.create_record.assert_called_with(type='TXT', + self.provider_mock.create_record.assert_called_with(rtype='TXT', name=self.record_name, content=self.record_content) - def test_add_txt_record_try_twice_to_find_domain(self): + def test_add_txt_record_try_twice_to_find_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, ''] self.client.add_txt_record(DOMAIN, self.record_name, self.record_content) - self.provider_mock.create_record.assert_called_with(type='TXT', + self.provider_mock.create_record.assert_called_with(rtype='TXT', name=self.record_name, content=self.record_content) - def test_add_txt_record_fail_to_find_domain(self): + def test_add_txt_record_fail_to_find_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND,] @@ -69,64 +116,64 @@ class BaseLexiconClientTest: self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_fail_to_authenticate(self): + def test_add_txt_record_fail_to_authenticate(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.LOGIN_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_fail_to_authenticate_with_unknown_error(self): + def test_add_txt_record_fail_to_authenticate_with_unknown_error(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.UNKNOWN_LOGIN_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_error_finding_domain(self): + def test_add_txt_record_error_finding_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.GENERIC_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_error_adding_record(self): + def test_add_txt_record_error_adding_record(self: _LexiconAwareTestCase): self.provider_mock.create_record.side_effect = self.GENERIC_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_del_txt_record(self): + def test_del_txt_record(self: _LexiconAwareTestCase): self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - self.provider_mock.delete_record.assert_called_with(type='TXT', + self.provider_mock.delete_record.assert_called_with(rtype='TXT', name=self.record_name, content=self.record_content) - def test_del_txt_record_fail_to_find_domain(self): + def test_del_txt_record_fail_to_find_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND, ] self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_fail_to_authenticate(self): + def test_del_txt_record_fail_to_authenticate(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.LOGIN_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_fail_to_authenticate_with_unknown_error(self): + def test_del_txt_record_fail_to_authenticate_with_unknown_error(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.UNKNOWN_LOGIN_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_error_finding_domain(self): + def test_del_txt_record_error_finding_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.GENERIC_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_error_deleting_record(self): + def test_del_txt_record_error_deleting_record(self: _LexiconAwareTestCase): self.provider_mock.delete_record.side_effect = self.GENERIC_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index 0d32d6850..602c62d1e 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -26,8 +26,8 @@ class PluginStorage: self._config = config self._classkey = classkey self._initialized = False - self._data = None - self._storagepath = None + self._data: Dict + self._storagepath: str def _initialize_storage(self): """Initializes PluginStorage data and reads current state from the disk diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index e6777a7da..ebc69fcbe 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -6,7 +6,6 @@ import shutil import time import traceback - from certbot import errors from certbot import util from certbot._internal import constants diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 78236fa06..b892ad0a1 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -27,6 +27,9 @@ from certbot.compat import os from certbot.display import util as display_util try: + # When we remove this deprecated import, we should also remove the + # "external-mock" test environment and the mock dependency listed in + # tools/pinning/pyproject.toml. import mock warnings.warn( "The external mock module is being used for backwards compatibility " @@ -331,7 +334,7 @@ class TempDirTestCase(unittest.TestCase): class ConfigTestCase(TempDirTestCase): """Test class which sets up a NamespaceConfig object.""" def setUp(self): - super(ConfigTestCase, self).setUp() + super().setUp() self.config = configuration.NamespaceConfig( mock.MagicMock(**constants.CLI_DEFAULTS) ) diff --git a/certbot/docs/ciphers.rst b/certbot/docs/ciphers.rst index 43f648898..b4ef5902a 100644 --- a/certbot/docs/ciphers.rst +++ b/certbot/docs/ciphers.rst @@ -1,3 +1,11 @@ +.. + Sphinx complains that this file isn't included in any toctree, however, we + currently link to it in the section about installing Certbot through Docker. + Setting :orphan: below suppresses this warning. See + https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html#special-metadata-fields. + +:orphan: + ============ Ciphersuites ============ diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index 27511777b..5d772d90e 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -118,12 +118,12 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.13.0 - (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX - Installer/YYY (SUBCOMMAND; flags: FLAGS) - Py/major.minor.patchlevel). The flags encoded in the - user agent are: --duplicate, --force-renew, --allow- - subset-of-names, -n, and whether any hooks are set. + "". (default: CertbotACMEClient/1.15.0 (certbot; + OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). + The flags encoded in the user agent are: --duplicate, + --force-renew, --allow-subset-of-names, -n, and + whether any hooks are set. --user-agent-comment USER_AGENT_COMMENT Add a comment to the default user agent string. May be used when repackaging Certbot or calling it from @@ -216,9 +216,7 @@ testing: (invalid) certificates; equivalent to --server https://acme-staging-v02.api.letsencrypt.org/directory (default: False) - --debug Show tracebacks in case of errors, and allow certbot- - auto execution on experimental platforms (default: - False) + --debug Show tracebacks in case of errors (default: False) --no-verify-ssl Disable verification of the ACME server's certificate. (default: False) --http-01-port HTTP01_PORT diff --git a/certbot/docs/conf.py b/certbot/docs/conf.py index 254bd3edd..5496c42a2 100644 --- a/certbot/docs/conf.py +++ b/certbot/docs/conf.py @@ -98,7 +98,6 @@ language = None exclude_patterns = [ '_build', 'challenges.rst', - 'ciphers.rst' ] # The reST default role (used for this markup: `text`) to use for all diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 13ac5ed68..94e35f1bf 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -222,8 +222,6 @@ certbot-apache and certbot-nginx client code to configure specific web servers certbot-dns-* client code to configure DNS providers -certbot-auto and letsencrypt-auto - shell scripts to install Certbot and its dependencies on UNIX systems windows installer Installs Certbot on Windows and is built using the files in windows-installer/ @@ -548,53 +546,6 @@ Instructions for how to manually build and run the Certbot snap and the external snapped DNS plugins that the Certbot project supplies are located in the README file at https://github.com/certbot/certbot/tree/master/tools/snap. -Updating certbot-auto and letsencrypt-auto -========================================== - -.. note:: We are currently only accepting changes to certbot-auto that fix - regressions on platforms where certbot-auto is the recommended installation - method at https://certbot.eff.org/instructions. If you are unsure if a change - you want to make qualifies, don't hesitate to `ask for help`_! - -Updating the scripts --------------------- -Developers should *not* modify the ``certbot-auto`` and ``letsencrypt-auto`` files -in the root directory of the repository. Rather, modify the -``letsencrypt-auto.template`` and associated platform-specific shell scripts in -the ``letsencrypt-auto-source`` and -``letsencrypt-auto-source/pieces/bootstrappers`` directory, respectively. - -Building letsencrypt-auto-source/letsencrypt-auto -------------------------------------------------- -Once changes to any of the aforementioned files have been made, the -``letsencrypt-auto-source/letsencrypt-auto`` script should be updated. In lieu of -manually updating this script, run the build script, which lives at -``letsencrypt-auto-source/build.py``: - -.. code-block:: shell - - python letsencrypt-auto-source/build.py - -Running ``build.py`` will update the ``letsencrypt-auto-source/letsencrypt-auto`` -script. Note that the ``certbot-auto`` and ``letsencrypt-auto`` scripts in the root -directory of the repository will remain **unchanged** after this script is run. -Your changes will be propagated to these files during the next release of -Certbot. - -Opening a PR ------------- -When opening a PR, ensure that the following files are committed: - -1. ``letsencrypt-auto-source/letsencrypt-auto.template`` and - ``letsencrypt-auto-source/pieces/bootstrappers/*`` -2. ``letsencrypt-auto-source/letsencrypt-auto`` (generated by ``build.py``) - -It might also be a good idea to double check that **no** changes were -inadvertently made to the ``certbot-auto`` or ``letsencrypt-auto`` scripts in the -root of the repository. These scripts will be updated by the core developers -during the next release. - - Updating the documentation ========================== diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index ac6c798eb..4533cfcc1 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -125,117 +125,6 @@ of the ``/etc/letsencrypt`` directory, see :ref:`where-certs`. .. _Docker: https://docker.com .. _`install Docker`: https://docs.docker.com/engine/installation/ -Operating System Packages -------------------------- - -.. warning:: While the Certbot team tries to keep the Certbot packages offered - by various operating systems working in the most basic sense, due to - distribution policies and/or the limited resources of distribution - maintainers, Certbot OS packages often have problems that other distribution - mechanisms do not. The packages are often old resulting in a lack of bug - fixes and features and a worse TLS configuration than is generated by newer - versions of Certbot. They also may not configure certificate renewal for you - or have all of Certbot's plugins available. For reasons like these, we - recommend most users follow the instructions at - https://certbot.eff.org/instructions and OS packages are only documented - here as an alternative. - -**Arch Linux** - -.. code-block:: shell - - sudo pacman -S certbot - -**Debian** - -If you run Debian Buster or Debian testing/Sid, you can easily install certbot -packages through commands like: - -.. code-block:: shell - - sudo apt-get update - sudo apt-get install certbot - -If you run Debian Stretch, we recommend you use the packages in Debian -backports repository. First you'll have to follow the instructions at -https://backports.debian.org/Instructions/ to enable the Stretch backports repo, -if you have not already done so. Then run: - -.. code-block:: shell - - sudo apt-get install certbot -t stretch-backports - -In all of these cases, there also packages available to help Certbot integrate -with Apache, nginx, or various DNS services. If you are using Apache or nginx, -we strongly recommend that you install the ``python-certbot-apache`` or -``python-certbot-nginx`` package so that Certbot can fully automate HTTPS -configuration for your server. A full list of these packages can be found -through a command like: - -.. code-block:: shell - - apt search 'python-certbot*' - -They can be installed by running the same installation command above but -replacing ``certbot`` with the name of the desired package. - -**Ubuntu** - -If you run Ubuntu, certbot can be installed using: - -.. code-block:: shell - - sudo apt-get install certbot - -Optionally to install the Certbot Apache plugin, you can use: - -.. code-block:: shell - - sudo apt-get install python3-certbot-apache - -**Fedora** - -.. code-block:: shell - - sudo dnf install certbot python3-certbot-apache - -**FreeBSD** - - * Port: ``cd /usr/ports/security/py-certbot && make install clean`` - * Package: ``pkg install py37-certbot`` - -**Gentoo** - -The official Certbot client is available in Gentoo Portage. From the -official Certbot plugins, three of them are also available in Portage. -They need to be installed separately if you require their functionality. - -.. code-block:: shell - - emerge -av app-crypt/certbot - emerge -av app-crypt/certbot-apache - emerge -av app-crypt/certbot-nginx - emerge -av app-crypt/certbot-dns-nsone - -.. Note:: The ``app-crypt/certbot-dns-nsone`` package has a different - maintainer than the other packages and can lag behind in version. - -**NetBSD** - - * Build from source: ``cd /usr/pkgsrc/security/py-certbot && make install clean`` - * Install pre-compiled package: ``pkg_add py37-certbot`` - -**OpenBSD** - - * Port: ``cd /usr/ports/security/letsencrypt/client && make install clean`` - * Package: ``pkg_add letsencrypt`` - -**Other Operating Systems** - -OS packaging is an ongoing effort. If you'd like to package -Certbot for your distribution of choice please have a -look at the :doc:`packaging`. - .. _certbot-auto: Certbot-Auto @@ -251,25 +140,6 @@ Certbot on UNIX operating systems, however, this script is no longer supported. If you want to uninstall ``certbot-auto``, you can follow our instructions :doc:`here `. -Problems with Python virtual environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When using ``certbot-auto`` on a low memory system such as VPS with less than -512MB of RAM, the required dependencies of Certbot may fail to build. This can -be identified if the pip outputs contains something like ``internal compiler -error: Killed (program cc1)``. You can workaround this restriction by creating -a temporary swapfile:: - - user@webserver:~$ sudo fallocate -l 1G /tmp/swapfile - user@webserver:~$ sudo chmod 600 /tmp/swapfile - user@webserver:~$ sudo mkswap /tmp/swapfile - user@webserver:~$ sudo swapon /tmp/swapfile - -Disable and remove the swapfile once the virtual environment is constructed:: - - user@webserver:~$ sudo swapoff /tmp/swapfile - user@webserver:~$ sudo rm /tmp/swapfile - Pip --- diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 87c4b9569..be4d96c4f 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -14,7 +14,7 @@ obtaining, renewing, or revoking certificates. The most important and commonly-used commands will be discussed throughout this document; an exhaustive list also appears near the end of the document. -The ``certbot`` script on your web server might be named ``letsencrypt`` if your system uses an older package, or ``certbot-auto`` if you used an alternate installation method. Throughout the docs, whenever you see ``certbot``, swap in the correct name as needed. +The ``certbot`` script on your web server might be named ``letsencrypt`` if your system uses an older package, or ``certbot-auto`` if you used a now-deprecated installation method. Throughout the docs, whenever you see ``certbot``, swap in the correct name as needed. .. _plugins: @@ -284,6 +284,8 @@ dns-ispconfig_ Y N DNS Authentication using ISPConfig as DNS server dns-clouddns_ Y N DNS Authentication using CloudDNS API dns-lightsail_ Y N DNS Authentication using Amazon Lightsail DNS API dns-inwx_ Y Y DNS Authentication for INWX through the XML API +dns-azure_ Y N DNS Authentication using Azure DNS +dns-godaddy_ Y N DNS Authentication using Godaddy DNS ================== ==== ==== =============================================================== .. _haproxy: https://github.com/greenhost/certbot-haproxy @@ -298,6 +300,8 @@ dns-inwx_ Y Y DNS Authentication for INWX through the XML API .. _dns-clouddns: https://github.com/vshosting/certbot-dns-clouddns .. _dns-lightsail: https://github.com/noi/certbot-dns-lightsail .. _dns-inwx: https://github.com/oGGy990/certbot-dns-inwx/ +.. _dns-azure: https://github.com/binkhq/certbot-dns-azure +.. _dns-godaddy: https://github.com/miigotu/certbot-dns-godaddy If you're interested, you can also :ref:`write your own plugin `. @@ -420,7 +424,7 @@ option to control the curve used in ECDSA certificates. .. warning:: If you obtain certificates using ECDSA keys, you should be careful not to downgrade your Certbot installation since ECDSA keys are not supported by older versions of Certbot. Downgrades like this are possible if - you switch from something like the snaps or certbot-auto to packages + you switch from something like the snaps or pip to packages provided by your operating system which often lag behind. Changing existing certificates from RSA to ECDSA @@ -515,11 +519,8 @@ Renewing certificates days). Make sure you renew the certificates at least once in 3 months. -.. seealso:: Many of the certbot clients obtained through a - distribution come with automatic renewal out of the box, - such as Debian and Ubuntu versions installed through `apt`, - CentOS/RHEL 7 through EPEL, etc. See `Automated Renewals`_ - for more details. +.. seealso:: Most Certbot installations come with automatic + renewal out of the box. See `Automated Renewals`_ for more details. As of version 0.10.0, Certbot supports a ``renew`` action to check all installed certificates for impending expiry and attempt to renew @@ -689,27 +690,15 @@ The following commands could be used to specify where these files are located:: Automated Renewals ------------------ -Many Linux distributions provide automated renewal when you use the -packages installed through their system package manager. The -following table is an *incomplete* list of distributions which do so, -as well as their methods for doing so. +Most Certbot installations come with automatic renewals preconfigured. This +is done by means of a scheduled task which runs ``certbot renew`` periodically. -If you are not sure whether or not your system has this already -automated, refer to your distribution's documentation, or check your -system's crontab (typically in `/etc/crontab/` and `/etc/cron.*/*` and -systemd timers (`systemctl list-timers`). +If you are unsure whether you need to configure automated renewal: -.. csv-table:: Distributions with Automated Renewal - :header: "Distribution Name", "Distribution Version", "Automation Method" - - "CentOS", "EPEL 7", "systemd" - "Debian", "stretch", "cron, systemd" - "Debian", "testing/sid", "cron, systemd" - "Fedora", "26", "systemd" - "Fedora", "27", "systemd" - "RHEL", "EPEL 7", "systemd" - "Ubuntu", "17.10", "cron, systemd" - "Ubuntu", "certbot PPA", "cron, systemd" +1. Review the instructions for your system at https://certbot.eff.org/instructions. + They will describe how to set up a scheduled task, if necessary. +2. (Linux/BSD): Check your system's crontab (typically `/etc/crontab` and + `/etc/cron.*/*`) and systemd timers (``systemctl list-timers``). .. _where-certs: diff --git a/certbot/setup.py b/certbot/setup.py index 8b2874f20..e0078bd6e 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -8,6 +8,12 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup +min_setuptools_version='39.0.1' +# This conditional isn't necessary, but it provides better error messages to +# people who try to install this package with older versions of setuptools. +if LooseVersion(setuptools_version) < LooseVersion(min_setuptools_version): + raise RuntimeError(f'setuptools {min_setuptools_version}+ is required') + # Workaround for https://bugs.python.org/issue8876, see # https://bugs.python.org/issue8876#msg208792 # This can be removed when using Python 2.7.9 or later: @@ -49,29 +55,14 @@ install_requires = [ 'parsedatetime>=2.4', 'pyrfc3339', 'pytz', - 'setuptools>=39.0.1', + # This dependency needs to be added using environment markers to avoid its + # installation on Linux. + 'pywin32>=300 ; sys_platform == "win32"', + f'setuptools>={min_setuptools_version}', 'zope.component', 'zope.interface', ] -# Add pywin32 on Windows platforms to handle low-level system calls. -# This dependency needs to be added using environment markers to avoid its installation on Linux. -# However environment markers are supported only with setuptools >= 36.2. -# So this dependency is not added for old Linux distributions with old setuptools, -# in order to allow these systems to build certbot from sources. -pywin32_req = 'pywin32>=300' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append(pywin32_req + " ; sys_platform == 'win32'") -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif os.name == 'nt': - # This branch exists to improve this package's behavior on Windows. Without - # it, if the sdist is installed on Windows with an old version of - # setuptools, pywin32 will not be specified as a dependency. - install_requires.append(pywin32_req) - dev_extras = [ 'astroid', 'azure-devops', @@ -79,10 +70,16 @@ dev_extras = [ 'ipdb', 'mypy', 'PyGithub', + # 1.1.0+ is required for poetry to use the poetry-core library for the + # build system declared in tools/pinning/pyproject.toml. + 'poetry>=1.1.0', 'pylint', 'pytest', 'pytest-cov', 'pytest-xdist', + # typing-extensions is required to import typing.Protocol and make the mypy checks + # pass (along with pylint about non-existent objects) on Python 3.6 & 3.7 + 'typing-extensions', 'tox', 'twine', 'wheel', @@ -103,7 +100,7 @@ setup( long_description=readme, url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index d0448a1db..e034c5f32 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -106,7 +106,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): """Tests for certbot._internal.account.AccountFileStorage.""" def setUp(self): - super(AccountFileStorageTest, self).setUp() + super().setUp() from certbot._internal.account import AccountFileStorage self.storage = AccountFileStorage(self.config) @@ -150,7 +150,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): path = os.path.join(self.config.accounts_dir, self.acc.id, "regr.json") with open(path, "r") as f: regr = json.load(f) - self.assertTrue("new_authzr_uri" in regr) + self.assertIn("new_authzr_uri", regr) def test_update_regr(self): self.storage.update_regr(self.acc, self.mock_client) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 8eb5d7702..1f798c2d8 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -106,9 +106,9 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_time.sleep.call_count, 2) # Retry-After header is 30 seconds, but at the time sleep is invoked, several # instructions are executed, and next pool is in less than 30 seconds. - self.assertTrue(mock_time.sleep.call_args_list[1][0][0] <= 30) + self.assertLessEqual(mock_time.sleep.call_args_list[1][0][0], 30) # However, assert that we did not took the default value of 3 seconds. - self.assertTrue(mock_time.sleep.call_args_list[1][0][0] > 3) + self.assertGreater(mock_time.sleep.call_args_list[1][0][0], 3) self.assertEqual(self.mock_auth.cleanup.call_count, 1) # Test if list first element is http-01, use typ because it is an achall @@ -139,7 +139,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_auth.cleanup.call_count, 1) # Test if list first element is http-01, use typ because it is an achall for achall in self.mock_auth.cleanup.call_args[0][0]: - self.assertTrue(achall.typ in ["http-01", "dns-01"]) + self.assertIn(achall.typ, ["http-01", "dns-01"]) # Length of authorizations list self.assertEqual(len(authzr), 1) @@ -225,7 +225,7 @@ class HandleAuthorizationsTest(unittest.TestCase): with self.assertRaises(errors.AuthorizationError) as error: # We retry only once, so retries will be exhausted before STATUS_VALID is returned. self.handler.handle_authorizations(mock_order, False, 1) - self.assertTrue('All authorizations were not finalized by the CA.' in str(error.exception)) + self.assertIn('All authorizations were not finalized by the CA.', str(error.exception)) def test_no_domains(self): mock_order = mock.MagicMock(authorizations=[]) @@ -305,7 +305,7 @@ class HandleAuthorizationsTest(unittest.TestCase): with test_util.patch_get_utility(): with self.assertRaises(errors.AuthorizationError) as error: self.handler.handle_authorizations(mock_order, False) - self.assertTrue('Some challenges have failed.' in str(error.exception)) + self.assertIn('Some challenges have failed.', str(error.exception)) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") @@ -341,7 +341,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.handler.handle_authorizations(mock_order, True) # Despite best_effort=True, process will fail because no authzr is valid. - self.assertTrue('All challenges have failed.' in str(error.exception)) + self.assertIn('All challenges have failed.', str(error.exception)) def test_validated_challenge_not_rerun(self): # With a pending challenge that is not supported by the plugin, we @@ -486,7 +486,7 @@ class ReportFailedAuthzrsTest(unittest.TestCase): } # Prevent future regressions if the error type changes - self.assertTrue(kwargs["error"].description is not None) + self.assertIsNotNone(kwargs["error"].description) http_01 = messages.ChallengeBody(**kwargs) @@ -511,7 +511,7 @@ class ReportFailedAuthzrsTest(unittest.TestCase): auth_handler._report_failed_authzrs([self.authzr1], 'key') call_list = mock_zope().add_message.call_args_list self.assertEqual(len(call_list), 1) - self.assertTrue("Domain: example.com\nType: tls\nDetail: detail" in call_list[0][0][0]) + self.assertIn("Domain: example.com\nType: tls\nDetail: detail", call_list[0][0][0]) @test_util.patch_get_utility() def test_different_errors_and_domains(self, mock_zope): diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index ba6cfddc3..3e8fb0de7 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -26,7 +26,7 @@ class BaseCertManagerTest(test_util.ConfigTestCase): """Base class for setting up Cert Manager tests. """ def setUp(self): - super(BaseCertManagerTest, self).setUp() + super().setUp() self.config.quiet = False filesystem.makedirs(self.config.renewal_configs_dir) @@ -211,7 +211,7 @@ class CertificatesTest(BaseCertManagerTest): def test_certificates_quiet(self, mock_utility, mock_logger): self.config.quiet = True self._certificates(self.config) - self.assertFalse(mock_utility.notification.called) + self.assertIs(mock_utility.notification.called, False) self.assertTrue(mock_logger.warning.called) #pylint: disable=no-member @mock.patch('certbot.crypto_util.verify_renewable_cert') @@ -224,7 +224,7 @@ class CertificatesTest(BaseCertManagerTest): mock_verifier.return_value = None mock_report.return_value = "" self._certificates(self.config) - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) self.assertTrue(mock_report.called) self.assertTrue(mock_utility.called) self.assertTrue(mock_renewable_cert.called) @@ -242,7 +242,7 @@ class CertificatesTest(BaseCertManagerTest): filesystem.makedirs(empty_config.renewal_configs_dir) self._certificates(empty_config) - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) self.assertTrue(mock_utility.called) shutil.rmtree(empty_tempdir) @@ -269,31 +269,34 @@ class CertificatesTest(BaseCertManagerTest): get_report = lambda: cert_manager._report_human_readable(mock_config, parsed_certs) out = get_report() - self.assertTrue("INVALID: EXPIRED" in out) + self.assertIn("INVALID: EXPIRED", out) cert.target_expiry += datetime.timedelta(hours=2) # pylint: disable=protected-access out = get_report() - self.assertTrue('1 hour(s)' in out or '2 hour(s)' in out) - self.assertTrue('VALID' in out and 'INVALID' not in out) + self.assertIs('1 hour' in out or '2 hour(s)' in out, True) + self.assertIn('VALID', out) + self.assertNotIn('INVALID', out) cert.target_expiry += datetime.timedelta(days=1) # pylint: disable=protected-access out = get_report() - self.assertTrue('1 day' in out) - self.assertFalse('under' in out) - self.assertTrue('VALID' in out and 'INVALID' not in out) + self.assertIn('1 day', out) + self.assertNotIn('under', out) + self.assertIn('VALID', out) + self.assertNotIn('INVALID', out) cert.target_expiry += datetime.timedelta(days=2) # pylint: disable=protected-access out = get_report() - self.assertTrue('3 days' in out) - self.assertTrue('VALID' in out and 'INVALID' not in out) + self.assertIn('3 days', out) + self.assertIn('VALID', out) + self.assertNotIn('INVALID', out) cert.is_test_cert = True mock_revoked.return_value = True out = get_report() - self.assertTrue('INVALID: TEST_CERT, REVOKED' in out) + self.assertIn('INVALID: TEST_CERT, REVOKED', out) cert = mock.MagicMock(lineagename="indescribable") cert.target_expiry = expiry @@ -353,7 +356,7 @@ class LineageForCertnameTest(BaseCertManagerTest): def test_no_match(self, mock_renewal_conf_file, mock_make_or_verify_dir): mock_renewal_conf_file.return_value = "other.com.conf" from certbot._internal import cert_manager - self.assertEqual(cert_manager.lineage_for_certname(self.config, "example.com"), None) + self.assertIsNone(cert_manager.lineage_for_certname(self.config, "example.com")) self.assertTrue(mock_make_or_verify_dir.called) @mock.patch('certbot.util.make_or_verify_dir') @@ -361,7 +364,7 @@ class LineageForCertnameTest(BaseCertManagerTest): def test_no_renewal_file(self, mock_renewal_conf_file, mock_make_or_verify_dir): mock_renewal_conf_file.side_effect = errors.CertStorageError() from certbot._internal import cert_manager - self.assertEqual(cert_manager.lineage_for_certname(self.config, "example.com"), None) + self.assertIsNone(cert_manager.lineage_for_certname(self.config, "example.com")) self.assertTrue(mock_make_or_verify_dir.called) @@ -388,7 +391,7 @@ class DomainsForCertnameTest(BaseCertManagerTest): def test_no_match(self, mock_renewal_conf_file, mock_make_or_verify_dir): mock_renewal_conf_file.return_value = "somefile.conf" from certbot._internal import cert_manager - self.assertEqual(cert_manager.domains_for_certname(self.config, "other.com"), None) + self.assertIsNone(cert_manager.domains_for_certname(self.config, "other.com")) self.assertTrue(mock_make_or_verify_dir.called) @@ -396,7 +399,7 @@ class RenameLineageTest(BaseCertManagerTest): """Tests for certbot._internal.cert_manager.rename_lineage""" def setUp(self): - super(RenameLineageTest, self).setUp() + super().setUp() self.config.certname = "example.org" self.config.new_certname = "after" @@ -450,7 +453,7 @@ class RenameLineageTest(BaseCertManagerTest): self._call(self.config) from certbot._internal import cert_manager updated_lineage = cert_manager.lineage_for_certname(self.config, self.config.new_certname) - self.assertTrue(updated_lineage is not None) + self.assertIsNotNone(updated_lineage) self.assertEqual(updated_lineage.lineagename, self.config.new_certname) @test_util.patch_get_utility() @@ -463,7 +466,7 @@ class RenameLineageTest(BaseCertManagerTest): self._call(self.config) from certbot._internal import cert_manager updated_lineage = cert_manager.lineage_for_certname(self.config, self.config.new_certname) - self.assertTrue(updated_lineage is not None) + self.assertIsNotNone(updated_lineage) self.assertEqual(updated_lineage.lineagename, self.config.new_certname) @test_util.patch_get_utility() @@ -483,7 +486,7 @@ class DuplicativeCertsTest(storage_test.BaseRenewableCertTest): """Test to avoid duplicate lineages.""" def setUp(self): - super(DuplicativeCertsTest, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() @@ -503,12 +506,12 @@ class DuplicativeCertsTest(storage_test.BaseRenewableCertTest): result = find_duplicative_certs( self.config, ['example.com', 'www.example.com']) self.assertTrue(result[0].configfile.filename.endswith('example.org.conf')) - self.assertEqual(result[1], None) + self.assertIsNone(result[1]) # Superset result = find_duplicative_certs( self.config, ['example.com', 'www.example.com', 'something.new']) - self.assertEqual(result[0], None) + self.assertIsNone(result[0]) self.assertTrue(result[1].configfile.filename.endswith('example.org.conf')) # Partial overlap doesn't count @@ -521,7 +524,7 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): """Tests for certbot._internal.cert_manager.cert_path_to_lineage""" def setUp(self): - super(CertPathToLineageTest, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', @@ -581,7 +584,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): archive dirs.""" # A test with real overlapping archive dirs can be found in tests/boulder_integration.sh def setUp(self): - super(MatchAndCheckOverlaps, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', @@ -629,8 +632,7 @@ class GetCertnameTest(unittest.TestCase): self.assertEqual( cert_manager.get_certnames( self.config, "verb", allow_multiple=False), ['example.com']) - self.assertTrue( - prompt in self.mock_get_utility().menu.call_args[0][0]) + self.assertIn(prompt, self.mock_get_utility().menu.call_args[0][0]) @mock.patch('certbot._internal.storage.renewal_conf_files') @mock.patch('certbot._internal.storage.lineagename_for_filename') @@ -671,8 +673,7 @@ class GetCertnameTest(unittest.TestCase): self.assertEqual( cert_manager.get_certnames( self.config, "verb", allow_multiple=True), ['example.com']) - self.assertTrue( - prompt in self.mock_get_utility().checklist.call_args[0][0]) + self.assertIn(prompt, self.mock_get_utility().checklist.call_args[0][0]) @mock.patch('certbot._internal.storage.renewal_conf_files') @mock.patch('certbot._internal.storage.lineagename_for_filename') diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index fca2b3e3e..8cab7a5b1 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -150,80 +150,79 @@ class ParseTest(unittest.TestCase): def test_help(self): self._help_output(['--help']) # assert SystemExit is raised here out = self._help_output(['--help', 'all']) - self.assertTrue("--configurator" in out) - self.assertTrue("how a certificate is deployed" in out) - self.assertTrue("--webroot-path" in out) - self.assertTrue("--text" not in out) - self.assertTrue("%s" not in out) - self.assertTrue("{0}" not in out) - self.assertTrue("--renew-hook" not in out) + self.assertIn("--configurator", out) + self.assertIn("how a certificate is deployed", out) + self.assertIn("--webroot-path", out) + self.assertNotIn("--text", out) + self.assertNotIn("%s", out) + self.assertNotIn("{0}", out) + self.assertNotIn("--renew-hook", out) out = self._help_output(['-h', 'nginx']) if "nginx" in PLUGINS: # may be false while building distributions without plugins - self.assertTrue("--nginx-ctl" in out) - self.assertTrue("--webroot-path" not in out) - self.assertTrue("--checkpoints" not in out) + self.assertIn("--nginx-ctl", out) + self.assertNotIn("--webroot-path", out) + self.assertNotIn("--checkpoints", out) out = self._help_output(['-h']) - self.assertTrue("letsencrypt-auto" not in out) # test cli.cli_command if "nginx" in PLUGINS: - self.assertTrue("Use the Nginx plugin" in out) + self.assertIn("Use the Nginx plugin", out) else: - self.assertTrue("(the certbot nginx plugin is not" in out) + self.assertIn("(the certbot nginx plugin is not", out) out = self._help_output(['--help', 'plugins']) - self.assertTrue("--webroot-path" not in out) - self.assertTrue("--prepare" in out) - self.assertTrue('"plugins" subcommand' in out) + self.assertNotIn("--webroot-path", out) + self.assertIn("--prepare", out) + self.assertIn('"plugins" subcommand', out) # test multiple topics out = self._help_output(['-h', 'renew']) - self.assertTrue("--keep" in out) + self.assertIn("--keep", out) out = self._help_output(['-h', 'automation']) - self.assertTrue("--keep" in out) + self.assertIn("--keep", out) out = self._help_output(['-h', 'revoke']) - self.assertTrue("--keep" not in out) + self.assertNotIn("--keep", out) out = self._help_output(['--help', 'install']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) out = self._help_output(['--help', 'revoke']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) - self.assertTrue("--reason" in out) - self.assertTrue("--delete-after-revoke" in out) - self.assertTrue("--no-delete-after-revoke" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) + self.assertIn("--reason", out) + self.assertIn("--delete-after-revoke", out) + self.assertIn("--no-delete-after-revoke", out) out = self._help_output(['-h', 'register']) - self.assertTrue("--cert-path" not in out) - self.assertTrue("--key-path" not in out) + self.assertNotIn("--cert-path", out) + self.assertNotIn("--key-path", out) out = self._help_output(['-h']) - self.assertTrue(cli.SHORT_USAGE in out) - self.assertTrue(cli.COMMAND_OVERVIEW[:100] in out) - self.assertTrue("%s" not in out) - self.assertTrue("{0}" not in out) + self.assertIn(cli.SHORT_USAGE, out) + self.assertIn(cli.COMMAND_OVERVIEW[:100], out) + self.assertNotIn("%s", out) + self.assertNotIn("{0}", out) def test_help_no_dashes(self): self._help_output(['help']) # assert SystemExit is raised here out = self._help_output(['help', 'all']) - self.assertTrue("--configurator" in out) - self.assertTrue("how a certificate is deployed" in out) - self.assertTrue("--webroot-path" in out) - self.assertTrue("--text" not in out) - self.assertTrue("%s" not in out) - self.assertTrue("{0}" not in out) + self.assertIn("--configurator", out) + self.assertIn("how a certificate is deployed", out) + self.assertIn("--webroot-path", out) + self.assertNotIn("--text", out) + self.assertNotIn("%s", out) + self.assertNotIn("{0}", out) out = self._help_output(['help', 'install']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) out = self._help_output(['help', 'revoke']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) def test_parse_domains(self): short_args = ['-d', 'example.com'] @@ -271,8 +270,8 @@ class ParseTest(unittest.TestCase): def test_must_staple_flag(self): short_args = ['--must-staple'] namespace = self.parse(short_args) - self.assertTrue(namespace.must_staple) - self.assertTrue(namespace.staple) + self.assertIs(namespace.must_staple, True) + self.assertIs(namespace.staple, True) def _check_server_conflict_message(self, parser_args, conflicting_args): try: @@ -281,31 +280,31 @@ class ParseTest(unittest.TestCase): "The following flags didn't conflict with " '--server: {0}'.format(', '.join(conflicting_args))) except errors.Error as error: - self.assertTrue('--server' in str(error)) + self.assertIn('--server', str(error)) for arg in conflicting_args: - self.assertTrue(arg in str(error)) + self.assertIn(arg, str(error)) def test_staging_flag(self): short_args = ['--staging'] namespace = self.parse(short_args) - self.assertTrue(namespace.staging) + self.assertIs(namespace.staging, True) self.assertEqual(namespace.server, constants.STAGING_URI) short_args += '--server example.com'.split() self._check_server_conflict_message(short_args, '--staging') def _assert_dry_run_flag_worked(self, namespace, existing_account): - self.assertTrue(namespace.dry_run) - self.assertTrue(namespace.break_my_certs) - self.assertTrue(namespace.staging) + self.assertIs(namespace.dry_run, True) + self.assertIs(namespace.break_my_certs, True) + self.assertIs(namespace.staging, True) self.assertEqual(namespace.server, constants.STAGING_URI) if existing_account: - self.assertTrue(namespace.tos) - self.assertTrue(namespace.register_unsafely_without_email) + self.assertIs(namespace.tos, True) + self.assertIs(namespace.register_unsafely_without_email, True) else: - self.assertFalse(namespace.tos) - self.assertFalse(namespace.register_unsafely_without_email) + self.assertIs(namespace.tos, False) + self.assertIs(namespace.register_unsafely_without_email, False) def test_dry_run_flag(self): config_dir = tempfile.mkdtemp() @@ -351,8 +350,8 @@ class ParseTest(unittest.TestCase): key_size_value = cli.flag_default(key_size_option) self.parse('--rsa-key-size {0}'.format(key_size_value).split()) - self.assertTrue(cli.option_was_set(key_size_option, key_size_value)) - self.assertTrue(cli.option_was_set('no_verify_ssl', True)) + self.assertIs(cli.option_was_set(key_size_option, key_size_value), True) + self.assertIs(cli.option_was_set('no_verify_ssl', True), True) config_dir_option = 'config_dir' self.assertFalse(cli.option_was_set( @@ -426,7 +425,7 @@ class ParseTest(unittest.TestCase): value = "foo" namespace = self.parse( ["--renew-hook", value, "--disable-hook-validation"]) - self.assertEqual(namespace.deploy_hook, None) + self.assertIsNone(namespace.deploy_hook) self.assertEqual(namespace.renew_hook, value) def test_max_log_backups_error(self): @@ -457,19 +456,19 @@ class ParseTest(unittest.TestCase): self.assertFalse(self.parse(["--no-directory-hooks"]).directory_hooks) def test_no_directory_hooks_unset(self): - self.assertTrue(self.parse([]).directory_hooks) + self.assertIs(self.parse([]).directory_hooks, True) def test_delete_after_revoke(self): namespace = self.parse(["--delete-after-revoke"]) - self.assertTrue(namespace.delete_after_revoke) + self.assertIs(namespace.delete_after_revoke, True) def test_delete_after_revoke_default(self): namespace = self.parse([]) - self.assertEqual(namespace.delete_after_revoke, None) + self.assertIsNone(namespace.delete_after_revoke) def test_no_delete_after_revoke(self): namespace = self.parse(["--no-delete-after-revoke"]) - self.assertFalse(namespace.delete_after_revoke) + self.assertIs(namespace.delete_after_revoke, False) def test_allow_subset_with_wildcard(self): self.assertRaises(errors.Error, self.parse, @@ -478,7 +477,7 @@ class ParseTest(unittest.TestCase): def test_route53_no_revert(self): for help_flag in ['-h', '--help']: for topic in ['all', 'plugins', 'dns-route53']: - self.assertFalse('certbot-route53:auth' in self._help_output([help_flag, topic])) + self.assertNotIn('certbot-route53:auth', self._help_output([help_flag, topic])) class DefaultTest(unittest.TestCase): @@ -491,8 +490,8 @@ class DefaultTest(unittest.TestCase): self.default2 = cli._Default() def test_boolean(self): - self.assertFalse(self.default1) - self.assertFalse(self.default2) + self.assertIs(bool(self.default1), False) + self.assertIs(bool(self.default2), False) def test_equality(self): self.assertEqual(self.default1, self.default2) @@ -515,7 +514,7 @@ class SetByCliTest(unittest.TestCase): def test_webroot_map(self): args = '-w /var/www/html -d example.com'.split() verb = 'renew' - self.assertTrue(_call_set_by_cli('webroot_map', args, verb)) + self.assertIs(_call_set_by_cli('webroot_map', args, verb), True) def _call_set_by_cli(var, args, verb): diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f058cb658..a89636a63 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -47,7 +47,6 @@ class DetermineUserAgentTest(test_util.ConfigTestCase): doc_value_check = self.assertNotIn real_value_check = self.assertIn - doc_value_check("certbot(-auto)", ua) doc_value_check("OS_NAME OS_VERSION", ua) doc_value_check("major.minor.patchlevel", ua) real_value_check(util.get_os_info_ua(), ua) @@ -58,7 +57,7 @@ class RegisterTest(test_util.ConfigTestCase): """Tests for certbot._internal.client.register.""" def setUp(self): - super(RegisterTest, self).setUp() + super().setUp() self.config.rsa_key_size = 1024 self.config.register_unsafely_without_email = False self.config.email = "alias@example.com" @@ -94,11 +93,11 @@ class RegisterTest(test_util.ConfigTestCase): with mock.patch("certbot._internal.eff.prepare_subscription") as mock_prepare: mock_client().new_account_and_tos.side_effect = errors.Error self.assertRaises(errors.Error, self._call) - self.assertFalse(mock_prepare.called) + self.assertIs(mock_prepare.called, False) mock_client().new_account_and_tos.side_effect = None self._call() - self.assertTrue(mock_prepare.called) + self.assertIs(mock_prepare.called, True) def test_it(self): with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client: @@ -118,7 +117,7 @@ class RegisterTest(test_util.ConfigTestCase): mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self._call() self.assertEqual(mock_get_email.call_count, 1) - self.assertTrue(mock_prepare.called) + self.assertIs(mock_prepare.called, True) def test_email_invalid_noninteractive(self): from acme import messages @@ -145,7 +144,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.dry_run = False self._call() mock_logger.debug.assert_called_once_with(mock.ANY) - self.assertTrue(mock_prepare.called) + self.assertIs(mock_prepare.called, True) @mock.patch("certbot._internal.client.display_ops.get_email") def test_dry_run_no_staging_account(self, mock_get_email): @@ -156,7 +155,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.dry_run = True self._call() # check Certbot did not ask the user to provide an email - self.assertFalse(mock_get_email.called) + self.assertIs(mock_get_email.called, False) # check Certbot created an account with no email. Contact should return empty self.assertFalse(mock_client().new_account_and_tos.call_args[0][0].contact) @@ -173,7 +172,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.eab_hmac_key = "J2OAqW4MHXsrHVa_PVg0Y-L_R4SYw0_aL1le6mfblbE" self._call() - self.assertTrue(mock_eab_from_data.called) + self.assertIs(mock_eab_from_data.called, True) def test_without_eab_arguments(self): with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client: @@ -185,7 +184,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.eab_hmac_key = None self._call() - self.assertFalse(mock_eab_from_data.called) + self.assertIs(mock_eab_from_data.called, False) def test_external_account_required_without_eab_arguments(self): with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client: @@ -210,14 +209,14 @@ class RegisterTest(test_util.ConfigTestCase): with mock.patch("certbot._internal.eff.handle_subscription") as mock_handle: mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(messages.Error, self._call) - self.assertFalse(mock_handle.called) + self.assertIs(mock_handle.called, False) class ClientTestCommon(test_util.ConfigTestCase): """Common base class for certbot._internal.client.Client tests.""" def setUp(self): - super(ClientTestCommon, self).setUp() + super().setUp() self.config.no_verify_ssl = False self.config.allow_subset_of_names = False @@ -236,7 +235,7 @@ class ClientTest(ClientTestCommon): """Tests for certbot._internal.client.Client.""" def setUp(self): - super(ClientTest, self).setUp() + super().setUp() self.config.allow_subset_of_names = False self.config.dry_run = False @@ -247,7 +246,7 @@ class ClientTest(ClientTestCommon): def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] - self.assertTrue(net.verify_ssl) + self.assertIs(net.verify_ssl, True) def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() @@ -582,7 +581,7 @@ class EnhanceConfigTest(ClientTestCommon): """Tests for certbot._internal.client.Client.enhance_config.""" def setUp(self): - super(EnhanceConfigTest, self).setUp() + super().setUp() self.config.hsts = False self.config.redirect = False @@ -609,7 +608,7 @@ class EnhanceConfigTest(ClientTestCommon): def test_already_exists_header(self, mock_log): self.config.hsts = True self._test_with_already_existing() - self.assertTrue(mock_log.warning.called) + self.assertIs(mock_log.warning.called, True) self.assertEqual(mock_log.warning.call_args[0][1], 'Strict-Transport-Security') @@ -617,7 +616,7 @@ class EnhanceConfigTest(ClientTestCommon): def test_already_exists_redirect(self, mock_log): self.config.redirect = True self._test_with_already_existing() - self.assertTrue(mock_log.warning.called) + self.assertIs(mock_log.warning.called, True) self.assertEqual(mock_log.warning.call_args[0][1], 'redirect') @@ -625,13 +624,13 @@ class EnhanceConfigTest(ClientTestCommon): def test_config_set_no_warning_redirect(self, mock_log): self.config.redirect = False self._test_with_already_existing() - self.assertFalse(mock_log.warning.called) + self.assertIs(mock_log.warning.called, False) @mock.patch("certbot._internal.client.logger") def test_no_warn_redirect(self, mock_log): self.config.redirect = None self._test_with_all_supported() - self.assertFalse(mock_log.warning.called) + self.assertIs(mock_log.warning.called, False) def test_no_ask_hsts(self): self.config.hsts = True @@ -684,7 +683,7 @@ class EnhanceConfigTest(ClientTestCommon): def _test_error_with_rollback(self): self._test_error() - self.assertTrue(self.client.installer.restart.called) + self.assertIs(self.client.installer.restart.called, True) def _test_error(self): self.config.redirect = True diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index ea48c9d1c..9aab49c34 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -34,7 +34,7 @@ ADMINS_SID = 'S-1-5-32-544' class WindowsChmodTests(TempDirTestCase): """Unit tests for Windows chmod function in filesystem module""" def setUp(self): - super(WindowsChmodTests, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_symlink_resolution(self): @@ -157,17 +157,17 @@ class UmaskTest(TempDirTestCase): try: dir1 = os.path.join(self.tempdir, 'probe1') filesystem.mkdir(dir1) - self.assertTrue(filesystem.check_mode(dir1, 0o755)) + self.assertIs(filesystem.check_mode(dir1, 0o755), True) filesystem.umask(0o077) dir2 = os.path.join(self.tempdir, 'dir2') filesystem.mkdir(dir2) - self.assertTrue(filesystem.check_mode(dir2, 0o700)) + self.assertIs(filesystem.check_mode(dir2, 0o700), True) dir3 = os.path.join(self.tempdir, 'dir3') filesystem.mkdir(dir3, mode=0o777) - self.assertTrue(filesystem.check_mode(dir3, 0o700)) + self.assertIs(filesystem.check_mode(dir3, 0o700), True) finally: filesystem.umask(previous_umask) @@ -177,17 +177,17 @@ class UmaskTest(TempDirTestCase): try: file1 = os.path.join(self.tempdir, 'probe1') UmaskTest._create_file(file1) - self.assertTrue(filesystem.check_mode(file1, 0o755)) + self.assertIs(filesystem.check_mode(file1, 0o755), True) filesystem.umask(0o077) file2 = os.path.join(self.tempdir, 'probe2') UmaskTest._create_file(file2) - self.assertTrue(filesystem.check_mode(file2, 0o700)) + self.assertIs(filesystem.check_mode(file2, 0o700), True) file3 = os.path.join(self.tempdir, 'probe3') UmaskTest._create_file(file3) - self.assertTrue(filesystem.check_mode(file3, 0o700)) + self.assertIs(filesystem.check_mode(file3, 0o700), True) finally: filesystem.umask(previous_umask) @@ -203,7 +203,7 @@ class UmaskTest(TempDirTestCase): class ComputePrivateKeyModeTest(TempDirTestCase): def setUp(self): - super(ComputePrivateKeyModeTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_compute_private_key_mode(self): @@ -351,7 +351,7 @@ class MakedirsTests(test_util.TempDirTestCase): class CopyOwnershipAndModeTest(test_util.TempDirTestCase): """Tests about copy_ownership_and_apply_mode, copy_ownership_and_mode and has_same_ownership""" def setUp(self): - super(CopyOwnershipAndModeTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') @@ -400,7 +400,7 @@ class CopyOwnershipAndModeTest(test_util.TempDirTestCase): util.safe_open(path1, 'w').close() util.safe_open(path2, 'w').close() - self.assertTrue(filesystem.has_same_ownership(path1, path2)) + self.assertIs(filesystem.has_same_ownership(path1, path2), True) @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') def test_copy_ownership_and_mode_windows(self): @@ -408,8 +408,8 @@ class CopyOwnershipAndModeTest(test_util.TempDirTestCase): dst = _create_probe(self.tempdir, name='dst') filesystem.chmod(src, 0o700) - self.assertTrue(filesystem.check_mode(src, 0o700)) - self.assertTrue(filesystem.check_mode(dst, 0o744)) + self.assertIs(filesystem.check_mode(src, 0o700), True) + self.assertIs(filesystem.check_mode(dst, 0o744), True) # Checking an actual change of owner is tricky during a unit test, since we do not know # if any user exists beside the current one. So we mock _copy_win_ownership. It's behavior @@ -418,24 +418,24 @@ class CopyOwnershipAndModeTest(test_util.TempDirTestCase): filesystem.copy_ownership_and_mode(src, dst) mock_copy_owner.assert_called_once_with(src, dst) - self.assertTrue(filesystem.check_mode(dst, 0o700)) + self.assertIs(filesystem.check_mode(dst, 0o700), True) class CheckPermissionsTest(test_util.TempDirTestCase): """Tests relative to functions that check modes.""" def setUp(self): - super(CheckPermissionsTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_check_mode(self): - self.assertTrue(filesystem.check_mode(self.probe_path, 0o744)) + self.assertIs(filesystem.check_mode(self.probe_path, 0o744), True) filesystem.chmod(self.probe_path, 0o700) self.assertFalse(filesystem.check_mode(self.probe_path, 0o744)) @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') def test_check_owner_windows(self): - self.assertTrue(filesystem.check_owner(self.probe_path)) + self.assertIs(filesystem.check_owner(self.probe_path), True) system = win32security.ConvertStringSidToSid(SYSTEM_SID) security = win32security.SECURITY_ATTRIBUTES().SECURITY_DESCRIPTOR @@ -447,7 +447,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): @unittest.skipUnless(POSIX_MODE, reason='Test specific to Linux security') def test_check_owner_linux(self): - self.assertTrue(filesystem.check_owner(self.probe_path)) + self.assertIs(filesystem.check_owner(self.probe_path), True) import os as std_os # pylint: disable=os-module-forbidden # See related inline comment in certbot.compat.filesystem.check_owner method @@ -459,7 +459,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): self.assertFalse(filesystem.check_owner(self.probe_path)) def test_check_permissions(self): - self.assertTrue(filesystem.check_permissions(self.probe_path, 0o744)) + self.assertIs(filesystem.check_permissions(self.probe_path, 0o744), True) with mock.patch('certbot.compat.filesystem.check_mode') as mock_mode: mock_mode.return_value = False @@ -471,7 +471,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_check_min_permissions(self): filesystem.chmod(self.probe_path, 0o744) - self.assertTrue(filesystem.has_min_permissions(self.probe_path, 0o744)) + self.assertIs(filesystem.has_min_permissions(self.probe_path, 0o744), True) filesystem.chmod(self.probe_path, 0o700) self.assertFalse(filesystem.has_min_permissions(self.probe_path, 0o744)) @@ -481,7 +481,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_is_world_reachable(self): filesystem.chmod(self.probe_path, 0o744) - self.assertTrue(filesystem.has_world_permissions(self.probe_path)) + self.assertIs(filesystem.has_world_permissions(self.probe_path), True) filesystem.chmod(self.probe_path, 0o700) self.assertFalse(filesystem.has_world_permissions(self.probe_path)) @@ -500,13 +500,13 @@ class OsReplaceTest(test_util.TempDirTestCase): filesystem.replace(src, dst) self.assertFalse(os.path.exists(src)) - self.assertTrue(os.path.exists(dst)) + self.assertIs(os.path.exists(dst), True) class RealpathTest(test_util.TempDirTestCase): """Tests for realpath method""" def setUp(self): - super(RealpathTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_symlink_resolution(self): @@ -542,7 +542,7 @@ class RealpathTest(test_util.TempDirTestCase): with self.assertRaises(RuntimeError) as error: filesystem.realpath(link1_path) - self.assertTrue('link1 is a loop!' in str(error.exception)) + self.assertIn('link1 is a loop!', str(error.exception)) class IsExecutableTest(test_util.TempDirTestCase): @@ -578,7 +578,7 @@ class IsExecutableTest(test_util.TempDirTestCase): with _fix_windows_runtime(): mock_access.return_value = True mock_isfile.return_value = True - self.assertTrue(filesystem.is_executable("/path/to/exe")) + self.assertIs(filesystem.is_executable("/path/to/exe"), True) @mock.patch("certbot.compat.filesystem.os.path.isfile") @mock.patch("certbot.compat.filesystem.os.access") @@ -586,7 +586,7 @@ class IsExecutableTest(test_util.TempDirTestCase): with _fix_windows_runtime(): mock_access.return_value = True mock_isfile.return_value = True - self.assertTrue(filesystem.is_executable("exe")) + self.assertIs(filesystem.is_executable("exe"), True) @mock.patch("certbot.compat.filesystem.os.path.isfile") @mock.patch("certbot.compat.filesystem.os.access") diff --git a/certbot/tests/compat/misc_test.py b/certbot/tests/compat/misc_test.py index 642f395ba..e87498cbe 100644 --- a/certbot/tests/compat/misc_test.py +++ b/certbot/tests/compat/misc_test.py @@ -45,4 +45,4 @@ class ExecuteTest(unittest.TestCase): mock_logger.info.assert_any_call(mock.ANY, mock.ANY, mock.ANY, stdout) if stderr or returncode: - self.assertTrue(mock_logger.error.called) + self.assertIs(mock_logger.error.called, True) diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index 1f8a0e803..e23c50afb 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -17,7 +17,7 @@ class NamespaceConfigTest(test_util.ConfigTestCase): """Tests for certbot._internal.configuration.NamespaceConfig.""" def setUp(self): - super(NamespaceConfigTest, self).setUp() + super().setUp() self.config.foo = 'bar' # pylint: disable=blacklisted-name self.config.server = 'https://acme-server.org:443/new' self.config.https_port = 1234 diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 7ff7bbeb9..043121dbb 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -36,7 +36,7 @@ CERT_ALT_ISSUER = test_util.load_vector('cert_intermediate_2.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" def setUp(self): - super(InitSaveKeyTest, self).setUp() + super().setUp() self.workdir = os.path.join(self.tempdir, 'workdir') filesystem.mkdir(self.workdir, mode=0o700) @@ -46,7 +46,7 @@ class InitSaveKeyTest(test_util.TempDirTestCase): mock.Mock(strict_permissions=True), interfaces.IConfig) def tearDown(self): - super(InitSaveKeyTest, self).tearDown() + super().tearDown() logging.disable(logging.NOTSET) @@ -60,7 +60,7 @@ class InitSaveKeyTest(test_util.TempDirTestCase): mock_make.return_value = b'key_pem' key = self._call(1024, self.workdir) self.assertEqual(key.pem, b'key_pem') - self.assertTrue('key-certbot.pem' in key.file) + self.assertIn('key-certbot.pem', key.file) self.assertTrue(os.path.exists(os.path.join(self.workdir, key.file))) @mock.patch('certbot.crypto_util.make_key') @@ -73,7 +73,7 @@ class InitSaveCSRTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_csr.""" def setUp(self): - super(InitSaveCSRTest, self).setUp() + super().setUp() zope.component.provideUtility( mock.Mock(strict_permissions=True), interfaces.IConfig) @@ -89,7 +89,7 @@ class InitSaveCSRTest(test_util.TempDirTestCase): mock.Mock(pem='dummy_key'), 'example.com', self.tempdir) self.assertEqual(csr.data, b'csr_pem') - self.assertTrue('csr-certbot.pem' in csr.file) + self.assertIn('csr-certbot.pem', csr.file) class ValidCSRTest(unittest.TestCase): @@ -251,7 +251,7 @@ class VerifyRenewableCertTest(VerifyCertSetup): return verify_renewable_cert(renewable_cert) def test_verify_renewable_cert(self): - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) @mock.patch('certbot.crypto_util.verify_renewable_cert_sig', side_effect=errors.Error("")) def test_verify_renewable_cert_failure(self, unused_verify_renewable_cert_sign): @@ -266,14 +266,14 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): return verify_renewable_cert_sig(renewable_cert) def test_cert_sig_match(self): - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) def test_cert_sig_match_ec(self): renewable_cert = mock.MagicMock() renewable_cert.cert_path = P256_CERT_PATH renewable_cert.chain_path = P256_CERT_PATH renewable_cert.key_path = P256_KEY - self.assertEqual(None, self._call(renewable_cert)) + self.assertIsNone(self._call(renewable_cert)) def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert_path = test_util.vector_path('cert_512_bad.pem') @@ -288,7 +288,7 @@ class VerifyFullchainTest(VerifyCertSetup): return verify_fullchain(renewable_cert) def test_fullchain_matches(self): - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) def test_fullchain_mismatch(self): self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) @@ -308,7 +308,7 @@ class VerifyCertMatchesPrivKeyTest(VerifyCertSetup): def test_cert_priv_key_match(self): self.renewable_cert.cert = SS_CERT_PATH self.renewable_cert.privkey = RSA2048_KEY_PATH - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) def test_cert_priv_key_mismatch(self): self.bad_renewable_cert.privkey = RSA256_KEY_PATH diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 4b08fe3a1..75b11d1d7 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -1,4 +1,6 @@ """Test certbot._internal.display.completer.""" +from typing import List + try: import readline # pylint: disable=import-error except ImportError: @@ -23,7 +25,7 @@ class CompleterTest(test_util.TempDirTestCase): """Test certbot._internal.display.completer.Completer.""" def setUp(self): - super(CompleterTest, self).setUp() + super().setUp() # directories must end with os.sep for completer to # search inside the directory for possible completions @@ -48,12 +50,12 @@ class CompleterTest(test_util.TempDirTestCase): for i in range(num_paths): completion = my_completer.complete(self.tempdir, i) - self.assertTrue(completion in self.paths) + self.assertIn(completion, self.paths) self.paths.remove(completion) - self.assertFalse(self.paths) + self.assertEqual(len(self.paths), 0) completion = my_completer.complete(self.tempdir, num_paths) - self.assertEqual(completion, None) + self.assertIsNone(completion) @unittest.skipIf('readline' not in sys.modules, reason='Not relevant if readline is not available.') @@ -96,7 +98,7 @@ class CompleterTest(test_util.TempDirTestCase): with completer.Completer(): pass - self.assertTrue(mock_readline.parse_and_bind.called) + self.assertIs(mock_readline.parse_and_bind.called, True) def enable_tab_completion(unused_command): diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 56f6415f6..7b828169c 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -61,9 +61,9 @@ class GetEmailTest(unittest.TestCase): with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.return_value = True self._call() - self.assertTrue(invalid_txt not in mock_input.call_args[0][0]) + self.assertNotIn(invalid_txt, mock_input.call_args[0][0]) self._call(invalid=True) - self.assertTrue(invalid_txt in mock_input.call_args[0][0]) + self.assertIn(invalid_txt, mock_input.call_args[0][0]) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_optional_flag(self, mock_get_utility): @@ -73,8 +73,7 @@ class GetEmailTest(unittest.TestCase): mock_safe_email.side_effect = [False, True] self._call(optional=False) for call in mock_input.call_args_list: - self.assertTrue( - "--register-unsafely-without-email" not in call[0][0]) + self.assertNotIn("--register-unsafely-without-email", call[0][0]) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_optional_invalid_unsafe(self, mock_get_utility): @@ -84,13 +83,13 @@ class GetEmailTest(unittest.TestCase): with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.side_effect = [False, True] self._call(invalid=True) - self.assertTrue(invalid_txt in mock_input.call_args[0][0]) + self.assertIn(invalid_txt, mock_input.call_args[0][0]) class ChooseAccountTest(test_util.TempDirTestCase): """Tests for certbot.display.ops.choose_account.""" def setUp(self): - super(ChooseAccountTest, self).setUp() + super().setUp() zope.component.provideUtility(display_util.FileDisplay(sys.stdout, False)) @@ -128,7 +127,7 @@ class ChooseAccountTest(test_util.TempDirTestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_cancel(self, mock_util): mock_util().menu.return_value = (display_util.CANCEL, 1) - self.assertTrue(self._call([self.acc1, self.acc2]) is None) + self.assertIsNone(self._call([self.acc1, self.acc2])) class GenHttpsNamesTest(unittest.TestCase): @@ -210,8 +209,7 @@ class ChooseNamesTest(unittest.TestCase): actual_doms = self._call(self.mock_install) self.assertEqual(mock_util().input.call_count, 1) self.assertEqual(actual_doms, [domain]) - self.assertTrue( - "configuration files" in mock_util().input.call_args[0][0]) + self.assertIn("configuration files", mock_util().input.call_args[0][0]) def test_sort_names_trivial(self): from certbot.display.ops import _sort_names @@ -354,7 +352,7 @@ class SuccessInstallationTest(unittest.TestCase): arg = mock_notify.call_args_list[0][0][0] for name in names: - self.assertTrue(name in arg) + self.assertIn(name, arg) class SuccessRenewalTest(unittest.TestCase): @@ -476,8 +474,8 @@ class ChooseValuesTest(unittest.TestCase): mock_util().checklist.return_value = (display_util.OK, [items[2]]) result = self._call(items, None) self.assertEqual(result, [items[2]]) - self.assertTrue(mock_util().checklist.called) - self.assertEqual(mock_util().checklist.call_args[0][0], None) + self.assertIs(mock_util().checklist.called, True) + self.assertIsNone(mock_util().checklist.call_args[0][0]) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_choose_names_success_question(self, mock_util): @@ -486,7 +484,7 @@ class ChooseValuesTest(unittest.TestCase): mock_util().checklist.return_value = (display_util.OK, [items[1]]) result = self._call(items, question) self.assertEqual(result, [items[1]]) - self.assertTrue(mock_util().checklist.called) + self.assertIs(mock_util().checklist.called, True) self.assertEqual(mock_util().checklist.call_args[0][0], question) @test_util.patch_get_utility("certbot.display.ops.z_util") @@ -496,7 +494,7 @@ class ChooseValuesTest(unittest.TestCase): mock_util().checklist.return_value = (display_util.CANCEL, []) result = self._call(items, question) self.assertEqual(result, []) - self.assertTrue(mock_util().checklist.called) + self.assertIs(mock_util().checklist.called, True) self.assertEqual(mock_util().checklist.call_args[0][0], question) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 30b33bbc9..ca7ecf908 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -65,7 +65,7 @@ class FileOutputDisplayTest(unittest.TestCase): """ def setUp(self): - super(FileOutputDisplayTest, self).setUp() + super().setUp() self.mock_stdout = mock.MagicMock() self.displayer = display_util.FileDisplay(self.mock_stdout, False) @@ -74,7 +74,7 @@ class FileOutputDisplayTest(unittest.TestCase): self.displayer.notification("message", False) string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) mock_logger.debug.assert_called_with("Notifying user: %s", "message") def test_notification_pause(self): @@ -82,25 +82,25 @@ class FileOutputDisplayTest(unittest.TestCase): with mock.patch(input_with_timeout, return_value="enter"): self.displayer.notification("message", force_interactive=True) - self.assertTrue("message" in self.mock_stdout.write.call_args[0][0]) + self.assertIn("message", self.mock_stdout.write.call_args[0][0]) def test_notification_noninteractive(self): self._force_noninteractive(self.displayer.notification, "message") string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) def test_notification_noninteractive2(self): # The main purpose of this test is to make sure we only call # logger.warning once which _force_noninteractive checks internally self._force_noninteractive(self.displayer.notification, "message") string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) self.assertTrue(self.displayer.skipped_interaction) self._force_noninteractive(self.displayer.notification, "message2") string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message2" in string) + self.assertIn("message2", string) def test_notification_decoration(self): from certbot.compat import os @@ -110,7 +110,8 @@ class FileOutputDisplayTest(unittest.TestCase): self.displayer.notification("message2", pause=False) string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("- - - " in string and ("message2" + os.linesep) in string) + self.assertIn("- - - ", string) + self.assertIn("message2" + os.linesep, string) @mock.patch("certbot.display.util." "FileDisplay._get_valid_int_ans") @@ -265,7 +266,7 @@ class FileOutputDisplayTest(unittest.TestCase): result = func(*args, **kwargs) if skipped_interaction: - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) else: self.assertEqual(mock_logger.warning.call_count, 1) @@ -331,7 +332,7 @@ class FileOutputDisplayTest(unittest.TestCase): # force_interactive to prevent workflow regressions. for name in interfaces.IDisplay.names(): arg_spec = inspect.getfullargspec(getattr(self.displayer, name)) - self.assertTrue("force_interactive" in arg_spec.args) + self.assertIn("force_interactive", arg_spec.args) class NoninteractiveDisplayTest(unittest.TestCase): @@ -345,7 +346,7 @@ class NoninteractiveDisplayTest(unittest.TestCase): self.displayer.notification("message", 10) string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) mock_logger.debug.assert_called_with("Notifying user: %s", "message") def test_notification_decoration(self): @@ -401,7 +402,7 @@ class NoninteractiveDisplayTest(unittest.TestCase): method = getattr(self.displayer, name) # asserts method accepts arbitrary keyword arguments result = inspect.getfullargspec(method).varkw - self.assertFalse(result is None) + self.assertIsNotNone(result) class SeparateListInputTest(unittest.TestCase): diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index dc6b6b944..0527d87d9 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -22,7 +22,7 @@ _KEY = josepy.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) class SubscriptionTest(test_util.ConfigTestCase): """Abstract class for subscription tests.""" def setUp(self): - super(SubscriptionTest, self).setUp() + super().setUp() self.account = account.Account( regr=messages.RegistrationResource( uri=None, body=messages.Registration(), @@ -50,7 +50,7 @@ class PrepareSubscriptionTest(SubscriptionTest): self._call() actual = mock_notify.call_args[0][0] expected_part = "because you didn't provide an e-mail address" - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) self.assertIsNone(self.account.meta.register_to_eff) @test_util.patch_get_utility() @@ -92,7 +92,7 @@ class PrepareSubscriptionTest(SubscriptionTest): call_args, call_kwargs = mock_get_utility().yesno.call_args actual = call_args[0] expected_part = 'Electronic Frontier Foundation' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) self.assertFalse(call_kwargs.get('default', True)) @@ -105,7 +105,7 @@ class HandleSubscriptionTest(SubscriptionTest): @mock.patch('certbot._internal.eff.subscribe') def test_no_subscribe(self, mock_subscribe): self._call() - self.assertFalse(mock_subscribe.called) + self.assertIs(mock_subscribe.called, False) @mock.patch('certbot._internal.eff.subscribe') def test_subscribe(self, mock_subscribe): @@ -140,7 +140,7 @@ class SubscribeTest(unittest.TestCase): self.assertEqual(call_args[0], constants.EFF_SUBSCRIBE_URI) data = call_kwargs.get('data') - self.assertFalse(data is None) + self.assertIsNotNone(data) self.assertEqual(data.get('email'), self.email) def test_bad_status(self): @@ -148,7 +148,7 @@ class SubscribeTest(unittest.TestCase): self._call() actual = self._get_reported_message() expected_part = 'because your e-mail address appears to be invalid.' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) def test_not_ok(self): self.response.ok = False @@ -156,21 +156,21 @@ class SubscribeTest(unittest.TestCase): self._call() actual = self._get_reported_message() unexpected_part = 'because' - self.assertFalse(unexpected_part in actual) + self.assertNotIn(unexpected_part, actual) def test_response_not_json(self): self.response.json.side_effect = ValueError() self._call() actual = self._get_reported_message() expected_part = 'problem' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) def test_response_json_missing_status_element(self): self.json.clear() self._call() actual = self._get_reported_message() expected_part = 'problem' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) def _get_reported_message(self): self.assertTrue(self.mock_notify.called) @@ -179,7 +179,7 @@ class SubscribeTest(unittest.TestCase): @test_util.patch_get_utility() def test_subscribe(self, mock_get_utility): self._call() - self.assertFalse(mock_get_utility.called) + self.assertIs(mock_get_utility.called, False) if __name__ == '__main__': diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 265a42023..ee4c2215d 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -3,6 +3,7 @@ import contextlib import signal import sys import unittest +from typing import Callable, Dict, Union try: import mock @@ -119,7 +120,7 @@ class ErrorHandlerTest(unittest.TestCase): sys.exit(0) except SystemExit: pass - self.assertFalse(self.init_func.called) + self.assertIs(self.init_func.called, False) def test_regular_exit(self): func = mock.MagicMock() @@ -135,7 +136,7 @@ class ExitHandlerTest(ErrorHandlerTest): def setUp(self): from certbot._internal import error_handler - super(ExitHandlerTest, self).setUp() + super().setUp() self.handler = error_handler.ExitHandler(self.init_func, *self.init_args, **self.init_kwargs) diff --git a/certbot/tests/helpful_test.py b/certbot/tests/helpful_test.py index 1a5c2bea6..0abe277bf 100644 --- a/certbot/tests/helpful_test.py +++ b/certbot/tests/helpful_test.py @@ -18,10 +18,10 @@ class TestScanningFlags(unittest.TestCase): arg_parser = HelpfulArgumentParser(['run'], {}) detected_flag = arg_parser.prescan_for_flag('--help', ['all', 'certonly']) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) detected_flag = arg_parser.prescan_for_flag('-h', ['all, certonly']) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) def test_prescan_unvalid_topic(self): arg_parser = HelpfulArgumentParser(['--help', 'all'], {}) @@ -30,7 +30,7 @@ class TestScanningFlags(unittest.TestCase): self.assertIs(detected_flag, True) detected_flag = arg_parser.prescan_for_flag('-h', arg_parser.help_topics) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) def test_prescan_valid_topic(self): arg_parser = HelpfulArgumentParser(['-h', 'all'], {}) @@ -39,7 +39,7 @@ class TestScanningFlags(unittest.TestCase): self.assertEqual(detected_flag, 'all') detected_flag = arg_parser.prescan_for_flag('--help', arg_parser.help_topics) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) class TestDetermineVerbs(unittest.TestCase): '''Tests for determine_verb methods of HelpfulArgumentParser''' @@ -90,7 +90,7 @@ class TestAdd(unittest.TestCase): metavar="EAB_KID", help="Key Identifier for External Account Binding") parsed_args = arg_parser.parser.parse_args(["--eab-kid", None]) - self.assertIs(parsed_args.eab_kid, None) + self.assertIsNone(parsed_args.eab_kid) self.assertTrue(hasattr(parsed_args, 'eab_kid')) @@ -115,7 +115,7 @@ class TestAddGroup(unittest.TestCase): self.assertTrue(arg_parser.groups["run"]) arg_parser.add_group("certonly", description="description of certonly") with self.assertRaises(KeyError): - self.assertFalse(arg_parser.groups["certonly"]) + self.assertIs(arg_parser.groups["certonly"], False) class TestParseArgsErrors(unittest.TestCase): diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index b0522bd52..bcef2e398 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -59,7 +59,7 @@ class ValidateHookTest(test_util.TempDirTestCase): @mock.patch("certbot._internal.hooks._prog") def test_unset(self, mock_prog): self._call(None, "foo") - self.assertFalse(mock_prog.called) + self.assertIs(mock_prog.called, False) class HookTest(test_util.ConfigTestCase): @@ -93,7 +93,7 @@ class PreHookTest(HookTest): return pre_hook(*args, **kwargs) def setUp(self): - super(PreHookTest, self).setUp() + super().setUp() self.config.pre_hook = "foo" filesystem.makedirs(self.config.renewal_pre_hooks_dir) @@ -106,7 +106,7 @@ class PreHookTest(HookTest): def tearDown(self): # Reset this value so it's unmodified for future tests self._reset_pre_hook_already() - super(PreHookTest, self).tearDown() + super().tearDown() def _reset_pre_hook_already(self): from certbot._internal.hooks import executed_pre_hooks @@ -132,8 +132,8 @@ class PreHookTest(HookTest): with mock.patch("certbot._internal.hooks.logger") as mock_logger: mock_execute = self._call_with_mock_execute(self.config) - self.assertFalse(mock_execute.called) - self.assertFalse(mock_logger.info.called) + self.assertIs(mock_execute.called, False) + self.assertIs(mock_logger.info.called, False) def test_renew_disabled_dir_hooks(self): self.config.directory_hooks = False @@ -158,7 +158,7 @@ class PreHookTest(HookTest): def _test_no_executions_common(self): with mock.patch("certbot._internal.hooks.logger") as mock_logger: mock_execute = self._call_with_mock_execute(self.config) - self.assertFalse(mock_execute.called) + self.assertIs(mock_execute.called, False) self.assertTrue(mock_logger.info.called) @@ -171,7 +171,7 @@ class PostHookTest(HookTest): return post_hook(*args, **kwargs) def setUp(self): - super(PostHookTest, self).setUp() + super().setUp() self.config.post_hook = "bar" filesystem.makedirs(self.config.renewal_post_hooks_dir) @@ -184,7 +184,7 @@ class PostHookTest(HookTest): def tearDown(self): # Reset this value so it's unmodified for future tests self._reset_post_hook_eventually() - super(PostHookTest, self).tearDown() + super().tearDown() def _reset_post_hook_eventually(self): from certbot._internal.hooks import post_hooks @@ -266,7 +266,7 @@ class RunSavedPostHooksTest(HookTest): return self._call_with_mock_execute(*args, **kwargs) def setUp(self): - super(RunSavedPostHooksTest, self).setUp() + super().setUp() self.eventually: List[str] = [] def test_empty(self): @@ -319,7 +319,7 @@ class RenewalHookTest(HookTest): return mock_execute def setUp(self): - super(RenewalHookTest, self).setUp() + super().setUp() self.vars_to_clear = set( var for var in ("RENEWED_DOMAINS", "RENEWED_LINEAGE",) if var not in os.environ) @@ -327,7 +327,7 @@ class RenewalHookTest(HookTest): def tearDown(self): for var in self.vars_to_clear: os.environ.pop(var, None) - super(RenewalHookTest, self).tearDown() + super().tearDown() class DeployHookTest(RenewalHookTest): @@ -344,7 +344,7 @@ class DeployHookTest(RenewalHookTest): self.config.dry_run = True mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) + self.assertIs(mock_execute.called, False) self.assertTrue(mock_logger.warning.called) @mock.patch("certbot._internal.hooks.logger") @@ -352,8 +352,8 @@ class DeployHookTest(RenewalHookTest): self.config.deploy_hook = None mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) - self.assertFalse(mock_logger.info.called) + self.assertIs(mock_execute.called, False) + self.assertIs(mock_logger.info.called, False) def test_success(self): domains = ["example.org", "example.net"] @@ -373,7 +373,7 @@ class RenewHookTest(RenewalHookTest): return renew_hook(*args, **kwargs) def setUp(self): - super(RenewHookTest, self).setUp() + super().setUp() self.config.renew_hook = "foo" filesystem.makedirs(self.config.renewal_deploy_hooks_dir) @@ -392,7 +392,7 @@ class RenewHookTest(RenewalHookTest): self.config.dry_run = True mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) + self.assertIs(mock_execute.called, False) self.assertEqual(mock_logger.warning.call_count, 2) def test_no_hooks(self): @@ -402,8 +402,8 @@ class RenewHookTest(RenewalHookTest): with mock.patch("certbot._internal.hooks.logger") as mock_logger: mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) - self.assertFalse(mock_logger.info.called) + self.assertIs(mock_execute.called, False) + self.assertIs(mock_logger.info.called, False) def test_overlap(self): self.config.renew_hook = self.dir_hook diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 2f887d33e..b45eb8f7a 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -44,7 +44,7 @@ class LockFileTest(test_util.TempDirTestCase): return LockFile(*args, **kwargs) def setUp(self): - super(LockFileTest, self).setUp() + super().setUp() self.lock_path = os.path.join(self.tempdir, 'test.lock') def test_acquire_without_deletion(self): @@ -69,7 +69,7 @@ class LockFileTest(test_util.TempDirTestCase): try: locked_repr = repr(lock_file) self._test_repr_common(lock_file, locked_repr) - self.assertTrue('acquired' in locked_repr) + self.assertIn('acquired', locked_repr) finally: lock_file.release() @@ -78,11 +78,11 @@ class LockFileTest(test_util.TempDirTestCase): lock_file.release() released_repr = repr(lock_file) self._test_repr_common(lock_file, released_repr) - self.assertTrue('released' in released_repr) + self.assertIn('released', released_repr) def _test_repr_common(self, lock_file, lock_repr): - self.assertTrue(lock_file.__class__.__name__ in lock_repr) - self.assertTrue(self.lock_path in lock_repr) + self.assertIn(lock_file.__class__.__name__, lock_repr) + self.assertIn(self.lock_path, lock_repr) @test_util.skip_on_windows( 'Race conditions on lock are specific to the non-blocking file access approach on Linux.') @@ -102,7 +102,7 @@ class LockFileTest(test_util.TempDirTestCase): with mock.patch('certbot._internal.lock.filesystem.os.stat') as mock_stat: mock_stat.side_effect = delete_and_stat self._call(self.lock_path) - self.assertFalse(should_delete) + self.assertEqual(len(should_delete), 0) def test_removed(self): lock_file = self._call(self.lock_path) @@ -120,7 +120,7 @@ class LockFileTest(test_util.TempDirTestCase): try: self._call(self.lock_path) except IOError as err: - self.assertTrue(msg in str(err)) + self.assertIn(msg, str(err)) else: # pragma: no cover self.fail('IOError not raised') @@ -136,7 +136,7 @@ class LockFileTest(test_util.TempDirTestCase): try: self._call(self.lock_path) except OSError as err: - self.assertTrue(msg in str(err)) + self.assertIn(msg, str(err)) else: # pragma: no cover self.fail('OSError not raised') diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 33d89c3ea..12daf0000 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -5,7 +5,7 @@ import logging.handlers import sys import time import unittest - +from typing import Optional from acme import messages from certbot import errors @@ -28,7 +28,8 @@ class PreArgParseSetupTest(unittest.TestCase): @classmethod def _call(cls, *args, **kwargs): # pylint: disable=unused-argument from certbot._internal.log import pre_arg_parse_setup - return pre_arg_parse_setup() + with mock.patch('builtins.open', mock.mock_open()): + return pre_arg_parse_setup() @mock.patch('certbot._internal.log.sys') @mock.patch('certbot._internal.log.pre_arg_parse_except_hook') @@ -50,9 +51,8 @@ class PreArgParseSetupTest(unittest.TestCase): memory_handler = handler target = memory_handler.target # type: ignore else: - self.assertTrue(isinstance(handler, logging.StreamHandler)) - self.assertTrue( - isinstance(target, logging.StreamHandler)) + self.assertIsInstance(handler, logging.StreamHandler) + self.assertIsInstance(target, logging.StreamHandler) mock_register.assert_called_once_with(logging.shutdown) mock_sys.excepthook(1, 2, 3) @@ -69,7 +69,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): return post_arg_parse_setup(*args, **kwargs) def setUp(self): - super(PostArgParseSetupTest, self).setUp() + super().setUp() self.config.debug = False self.config.max_log_backups = 1000 self.config.quiet = False @@ -90,7 +90,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): self.stream_handler.close() self.temp_handler.close() self.devnull.close() - super(PostArgParseSetupTest, self).tearDown() + super().tearDown() def test_common(self): with mock.patch('certbot._internal.log.logging.getLogger') as mock_get_logger: @@ -135,7 +135,7 @@ class SetupLogFileHandlerTest(test_util.ConfigTestCase): return setup_log_file_handler(*args, **kwargs) def setUp(self): - super(SetupLogFileHandlerTest, self).setUp() + super().setUp() self.config.max_log_backups = 42 @mock.patch('certbot._internal.main.logging.handlers.RotatingFileHandler') @@ -145,7 +145,7 @@ class SetupLogFileHandlerTest(test_util.ConfigTestCase): try: self._call(self.config, 'test.log', '%(message)s') except errors.Error as err: - self.assertTrue('--logs-dir' in str(err)) + self.assertIn('--logs-dir', str(err)) else: # pragma: no cover self.fail('Error not raised.') @@ -336,7 +336,7 @@ class PostArgParseExceptHookTest(unittest.TestCase): mock_logger, output = self._test_common(get_acme_error, debug=False) self._assert_exception_logged(mock_logger.debug, messages.Error) self._assert_quiet_output(mock_logger, output) - self.assertFalse(messages.ERROR_PREFIX in output) + self.assertNotIn(messages.ERROR_PREFIX, output) def test_other_error(self): exc_type = ValueError @@ -378,20 +378,20 @@ class PostArgParseExceptHookTest(unittest.TestCase): def _assert_exception_logged(self, log_func, exc_type): self.assertTrue(log_func.called) call_kwargs = log_func.call_args[1] - self.assertTrue('exc_info' in call_kwargs) + self.assertIn('exc_info', call_kwargs) actual_exc_info = call_kwargs['exc_info'] expected_exc_info = (exc_type, mock.ANY, mock.ANY) self.assertEqual(actual_exc_info, expected_exc_info) def _assert_logfile_output(self, output): - self.assertTrue('Please see the logfile' in output) - self.assertTrue(self.log_path in output) + self.assertIn('Please see the logfile', output) + self.assertIn(self.log_path, output) def _assert_quiet_output(self, mock_logger, output): - self.assertFalse(mock_logger.exception.called) + self.assertIs(mock_logger.exception.called, False) self.assertTrue(mock_logger.debug.called) - self.assertTrue(self.error_msg in output) + self.assertIn(self.error_msg, output) class ExitWithLogPathTest(test_util.TempDirTestCase): @@ -406,13 +406,13 @@ class ExitWithLogPathTest(test_util.TempDirTestCase): open(log_file, 'w').close() err_str = self._test_common(log_file) - self.assertTrue('logfiles' not in err_str) - self.assertTrue(log_file in err_str) + self.assertNotIn('logfiles', err_str) + self.assertIn(log_file, err_str) def test_log_dir(self): err_str = self._test_common(self.tempdir) - self.assertTrue('logfiles' in err_str) - self.assertTrue(self.tempdir in err_str) + self.assertIn('logfiles', err_str) + self.assertIn(self.tempdir, err_str) # pylint: disable=inconsistent-return-statements def _test_common(self, *args, **kwargs): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 816ceb536..3b6b34fff 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -12,6 +12,7 @@ import sys import tempfile import traceback import unittest +from typing import List import josepy as jose import pytz @@ -84,24 +85,24 @@ class TestHandleCerts(unittest.TestCase): mock_set.return_value = False with self.assertRaises(errors.Error) as raised: main._handle_unexpected_key_type_migration(config, cert) - self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + self.assertIn("Please provide both --cert-name and --key-type", str(raised.exception)) mock_set.side_effect = lambda var: var != "certname" with self.assertRaises(errors.Error) as raised: main._handle_unexpected_key_type_migration(config, cert) - self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + self.assertIn("Please provide both --cert-name and --key-type", str(raised.exception)) mock_set.side_effect = lambda var: var != "key_type" with self.assertRaises(errors.Error) as raised: main._handle_unexpected_key_type_migration(config, cert) - self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + self.assertIn("Please provide both --cert-name and --key-type", str(raised.exception)) class RunTest(test_util.ConfigTestCase): """Tests for certbot._internal.main.run.""" def setUp(self): - super(RunTest, self).setUp() + super().setUp() self.domain = 'example.org' patches = [ mock.patch('certbot._internal.main._get_and_save_cert'), @@ -196,7 +197,7 @@ class CertonlyTest(unittest.TestCase): self._call('certonly --webroot -d example.com'.split()) def _assert_no_pause(self, message, pause=True): # pylint: disable=unused-argument - self.assertFalse(pause) + self.assertIs(pause, False) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @mock.patch('certbot._internal.cert_manager.domains_for_certname') @@ -260,8 +261,7 @@ class FindDomainsOrCertnameTest(unittest.TestCase): mock_config = mock.Mock(domains=None, certname=None) mock_choose_names.return_value = "domainname" # pylint: disable=protected-access - self.assertEqual(main._find_domains_or_certname(mock_config, None), - ("domainname", None)) + self.assertEqual(main._find_domains_or_certname(mock_config, None), ("domainname", None)) @mock.patch('certbot.display.ops.choose_names') def test_no_results(self, mock_choose_names): @@ -275,15 +275,17 @@ class FindDomainsOrCertnameTest(unittest.TestCase): mock_config = mock.Mock(domains=None, certname="one.com") mock_domains.return_value = ["one.com", "two.com"] # pylint: disable=protected-access - self.assertEqual(main._find_domains_or_certname(mock_config, None), - (["one.com", "two.com"], "one.com")) + self.assertEqual( + main._find_domains_or_certname(mock_config, None), + (["one.com", "two.com"], "one.com") + ) class RevokeTest(test_util.TempDirTestCase): """Tests for certbot._internal.main.revoke.""" def setUp(self): - super(RevokeTest, self).setUp() + super().setUp() shutil.copy(CERT_PATH, self.tempdir) self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, 'cert_512.pem')) @@ -402,7 +404,7 @@ class RevokeTest(test_util.TempDirTestCase): mock_get_utility().yesno.return_value = False mock_delete_if_appropriate.return_value = False self._call() - self.assertFalse(mock_delete.called) + self.assertIs(mock_delete.called, False) class DeleteIfAppropriateTest(test_util.ConfigTestCase): """Tests for certbot._internal.main._delete_if_appropriate """ @@ -512,7 +514,7 @@ class DetermineAccountTest(test_util.ConfigTestCase): """Tests for certbot._internal.main._determine_account.""" def setUp(self): - super(DetermineAccountTest, self).setUp() + super().setUp() self.config.account = None self.config.email = None self.config.register_unsafely_without_email = False @@ -536,13 +538,13 @@ class DetermineAccountTest(test_util.ConfigTestCase): self.config.account = self.accs[1].id self.assertEqual((self.accs[1], None), self._call()) self.assertEqual(self.accs[1].id, self.config.account) - self.assertTrue(self.config.email is None) + self.assertIsNone(self.config.email) def test_single_account(self): self.account_storage.save(self.accs[0], self.mock_client) self.assertEqual((self.accs[0], None), self._call()) self.assertEqual(self.accs[0].id, self.config.account) - self.assertTrue(self.config.email is None) + self.assertIsNone(self.config.email) @mock.patch('certbot._internal.client.display_ops.choose_account') def test_multiple_accounts(self, mock_choose_accounts): @@ -553,7 +555,7 @@ class DetermineAccountTest(test_util.ConfigTestCase): self.assertEqual( set(mock_choose_accounts.call_args[0][0]), set(self.accs)) self.assertEqual(self.accs[1].id, self.config.account) - self.assertTrue(self.config.email is None) + self.assertIsNone(self.config.email) @mock.patch('certbot._internal.client.display_ops.get_email') @mock.patch('certbot._internal.main.display_util.notify') @@ -584,7 +586,7 @@ class MainTest(test_util.ConfigTestCase): """Tests for different commands.""" def setUp(self): - super(MainTest, self).setUp() + super().setUp() filesystem.mkdir(self.config.logs_dir) self.standard_args = ['--config-dir', self.config.config_dir, @@ -597,7 +599,7 @@ class MainTest(test_util.ConfigTestCase): # Reset globals in cli reload_module(cli) - super(MainTest, self).tearDown() + super().tearDown() def _call(self, args, stdout=None, mockisfile=False): """Run the cli with output streams, actual client and optionally @@ -651,7 +653,7 @@ class MainTest(test_util.ConfigTestCase): pass finally: output = toy_out.getvalue() or toy_err.getvalue() - self.assertTrue("certbot" in output, "Output is {0}".format(output)) + self.assertIn("certbot", output, "Output is {0}".format(output)) def _cli_missing_flag(self, args, message): "Ensure that a particular error raises a missing cli flag error containing message" @@ -661,8 +663,8 @@ class MainTest(test_util.ConfigTestCase): main.main(self.standard_args + args[:]) # NOTE: parser can alter its args! except errors.MissingCommandlineFlag as exc_: exc = exc_ - self.assertTrue(message in str(exc)) - self.assertTrue(exc is not None) + self.assertIn(message, str(exc)) + self.assertIsNotNone(exc) @mock.patch('certbot._internal.log.post_arg_parse_setup') def test_noninteractive(self, _): @@ -690,11 +692,11 @@ class MainTest(test_util.ConfigTestCase): self._call_no_clientmock(args) os_ver = util.get_os_info_ua() ua = acme_net.call_args[1]["user_agent"] - self.assertTrue(os_ver in ua) + self.assertIn(os_ver, ua) import platform plat = platform.platform() if "linux" in plat.lower(): - self.assertTrue(util.get_os_info_ua() in ua) + self.assertIn(util.get_os_info_ua(), ua) with mock.patch('certbot._internal.main.client.acme_client.ClientNetwork') as acme_net: ua = "bandersnatch" @@ -803,8 +805,8 @@ class MainTest(test_util.ConfigTestCase): # Sending nginx a non-existent conf dir will simulate misconfiguration # (we can only do that if certbot-nginx is actually present) ret, _, _, _ = self._call(args) - self.assertTrue("The nginx plugin is not working" in ret) - self.assertTrue("MisconfigurationError" in ret) + self.assertIn("The nginx plugin is not working", ret) + self.assertIn("MisconfigurationError", ret) self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably") @@ -813,7 +815,7 @@ class MainTest(test_util.ConfigTestCase): mock_gsc.return_value = mock.MagicMock() self._call(["certonly", "--manual", "-d", "foo.bar"]) unused_config, auth, unused_installer = mock_init.call_args[0] - self.assertTrue(isinstance(auth, manual.Authenticator)) + self.assertIsInstance(auth, manual.Authenticator) with mock.patch('certbot._internal.main.certonly') as mock_certonly: self._call(["auth", "--standalone"]) @@ -951,7 +953,7 @@ class MainTest(test_util.ConfigTestCase): self._call(['-a', 'bad_auth', 'certonly']) assert False, "Exception should have been raised" except errors.PluginSelectionError as e: - self.assertTrue('The requested bad_auth plugin does not appear' in str(e)) + self.assertIn('The requested bad_auth plugin does not appear', str(e)) def test_check_config_sanity_domain(self): # FQDN @@ -1003,7 +1005,6 @@ class MainTest(test_util.ConfigTestCase): args += '-d foo.bar -a standalone certonly'.split() self._call(args) - @mock.patch('certbot._internal.main._report_new_cert') @test_util.patch_get_utility() def test_certonly_dry_run_new_request_success(self, mock_get_utility, mock_report): @@ -1013,7 +1014,7 @@ class MainTest(test_util.ConfigTestCase): self.assertEqual( mock_client.obtain_and_enroll_certificate.call_count, 1) self.assertEqual(mock_report.call_count, 1) - self.assertEqual(mock_report.call_args[0][0].dry_run, True) + self.assertIs(mock_report.call_args[0][0].dry_run, True) # Asserts we don't suggest donating after a successful dry run self.assertEqual(mock_get_utility().add_message.call_count, 0) @@ -1038,9 +1039,9 @@ class MainTest(test_util.ConfigTestCase): self.assertEqual(mock_report.call_count, 1) self.assertIn(cert_path, mock_report.call_args[0][2]) self.assertIn(key_path, mock_report.call_args[0][3]) - self.assertTrue( - 'donate' in mock_get_utility().add_message.call_args[0][0]) - self.assertTrue(mock_subscription.called) + self.assertIn( + 'donate', mock_get_utility().add_message.call_args[0][0]) + self.assertIs(mock_subscription.called, True) @mock.patch('certbot._internal.eff.handle_subscription') def test_certonly_new_request_failure(self, mock_subscription): @@ -1048,7 +1049,7 @@ class MainTest(test_util.ConfigTestCase): mock_client.obtain_and_enroll_certificate.return_value = False self.assertRaises(errors.Error, self._certonly_new_request_common, mock_client) - self.assertFalse(mock_subscription.called) + self.assertIs(mock_subscription.called, False) def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, @@ -1123,7 +1124,7 @@ class MainTest(test_util.ConfigTestCase): finally: if log_out: with open(os.path.join(self.config.logs_dir, "letsencrypt.log")) as lf: - self.assertTrue(log_out in lf.read()) + self.assertIn(log_out, lf.read()) return mock_lineage, mock_get_utility, stdout @@ -1136,7 +1137,7 @@ class MainTest(test_util.ConfigTestCase): lineage.latest_common_version()) self.assertEqual(mock_report.call_count, 1) self.assertIn('fullchain.pem', mock_report.call_args[0][2]) - self.assertTrue('donate' in get_utility().add_message.call_args[0][0]) + self.assertIn('donate', get_utility().add_message.call_args[0][0]) @mock.patch('certbot._internal.main.display_util.notify') @mock.patch('certbot._internal.log.logging.handlers.RotatingFileHandler.doRollover') @@ -1203,8 +1204,8 @@ class MainTest(test_util.ConfigTestCase): expiry = datetime.datetime.now() + datetime.timedelta(days=90) _, _, stdout = self._test_renewal_common(False, extra_args=None, should_renew=False, args=['renew'], expiry_date=expiry) - self.assertTrue('No renewals were attempted.' in stdout.getvalue()) - self.assertTrue('The following certificates are not due for renewal yet:' in stdout.getvalue()) + self.assertIn('No renewals were attempted.', stdout.getvalue()) + self.assertIn('The following certificates are not due for renewal yet:', stdout.getvalue()) @mock.patch('certbot._internal.log.post_arg_parse_setup') def test_quiet_renew(self, _): @@ -1212,7 +1213,7 @@ class MainTest(test_util.ConfigTestCase): args = ["renew", "--dry-run"] _, _, stdout = self._test_renewal_common(True, [], args=args, should_renew=True) out = stdout.getvalue() - self.assertTrue("renew" in out) + self.assertIn("renew", out) args = ["renew", "--dry-run", "-q"] _, _, stdout = self._test_renewal_common(True, [], args=args, @@ -1278,7 +1279,7 @@ class MainTest(test_util.ConfigTestCase): if assert_oc_called: self.assertTrue(mock_renew_cert.called) else: - self.assertFalse(mock_renew_cert.called) + self.assertIs(mock_renew_cert.called, False) def test_renew_no_renewalparams(self): self._test_renew_common(assert_oc_called=False, error_expected=True) @@ -1357,7 +1358,7 @@ class MainTest(test_util.ConfigTestCase): args=['renew', '--post-hook', '{0} -c "print(\'hello world\');"' .format(sys.executable)]) - self.assertTrue('No hooks were run.' in stdout.getvalue()) + self.assertIn('No hooks were run.', stdout.getvalue()) @test_util.patch_get_utility() @mock.patch('certbot._internal.main._find_lineage_for_domains_and_certname') @@ -1368,8 +1369,8 @@ class MainTest(test_util.ConfigTestCase): mock_renewal.return_value = ('reinstall', mock.MagicMock()) mock_init.return_value = mock_client = mock.MagicMock() self._call(['-d', 'foo.bar', '-a', 'standalone', 'certonly']) - self.assertFalse(mock_client.obtain_certificate.called) - self.assertFalse(mock_client.obtain_and_enroll_certificate.called) + self.assertIs(mock_client.obtain_certificate.called, False) + self.assertIs(mock_client.obtain_and_enroll_certificate.called, False) self.assertEqual(mock_get_utility().add_message.call_count, 0) mock_report_new_cert.assert_not_called() #self.assertTrue('donate' not in mock_get_utility().add_message.call_args[0][0]) @@ -1401,7 +1402,7 @@ class MainTest(test_util.ConfigTestCase): self._call(args) if '--dry-run' in args: - self.assertFalse(mock_client.save_certificate.called) + self.assertIs(mock_client.save_certificate.called, False) else: mock_client.save_certificate.assert_called_once_with( certr, chain, cert_path, chain_path, full_path) @@ -1416,15 +1417,14 @@ class MainTest(test_util.ConfigTestCase): self.assertIn('cert_512.pem', mock_csr_report.call_args[0][1]) self.assertIsNone(mock_csr_report.call_args[0][2]) self.assertIn('fullchain.pem', mock_csr_report.call_args[0][3]) - self.assertTrue( - 'donate' in mock_get_utility().add_message.call_args[0][0]) - self.assertTrue(mock_subscription.called) + self.assertIn('donate', mock_get_utility().add_message.call_args[0][0]) + self.assertIs(mock_subscription.called, True) @mock.patch('certbot._internal.main._csr_report_new_cert') def test_certonly_csr_dry_run(self, mock_csr_report): _ = self._test_certonly_csr_common(['--dry-run']) self.assertEqual(mock_csr_report.call_count, 1) - self.assertEqual(mock_csr_report.call_args[0][0].dry_run, True) + self.assertIs(mock_csr_report.call_args[0][0].dry_run, True) @mock.patch('certbot._internal.main._delete_if_appropriate') @mock.patch('certbot._internal.main.client.acme_client') @@ -1479,7 +1479,7 @@ class MainTest(test_util.ConfigTestCase): mocked_account.AccountFileStorage.return_value = mocked_storage mocked_storage.find_all.return_value = ["an account"] x = self._call_no_clientmock(["register", "--email", "user@example.org"]) - self.assertTrue("There is an existing account" in x[0]) + self.assertIn("There is an existing account", x[0]) @mock.patch('certbot._internal.plugins.selection.choose_configurator_plugins') @mock.patch('certbot._internal.updater._run_updaters') @@ -1492,7 +1492,7 @@ class MainTest(test_util.ConfigTestCase): updater.run_generic_updaters(self.config, None, None) # Make sure we're returning None, and hence not trying to run the # without installer - self.assertFalse(mock_run.called) + self.assertIs(mock_run.called, False) class UnregisterTest(unittest.TestCase): @@ -1536,7 +1536,7 @@ class UnregisterTest(unittest.TestCase): res = main.unregister(config, unused_plugins) - self.assertTrue(res is None) + self.assertIsNone(res) mock_notify.assert_called_once_with("Account deactivated.") def test_unregister_no_account(self): @@ -1553,7 +1553,7 @@ class UnregisterTest(unittest.TestCase): res = main.unregister(config, unused_plugins) m = "Could not find existing account to deactivate." self.assertEqual(res, m) - self.assertFalse(cb_client.acme.deactivate_registration.called) + self.assertIs(cb_client.acme.deactivate_registration.called, False) class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): @@ -1581,7 +1581,7 @@ class EnhanceTest(test_util.ConfigTestCase): """Tests for certbot._internal.main.enhance.""" def setUp(self): - super(EnhanceTest, self).setUp() + super().setUp() self.get_utility_patch = test_util.patch_get_utility() self.mock_get_utility = self.get_utility_patch.start() self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) @@ -1617,7 +1617,7 @@ class EnhanceTest(test_util.ConfigTestCase): self._call(['enhance', '--redirect']) self.assertTrue(mock_pick.called) # Check that the message includes "enhancements" - self.assertTrue("enhancements" in mock_pick.call_args[0][3]) + self.assertIn("enhancements", mock_pick.call_args[0][3]) @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins') @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @@ -1631,7 +1631,7 @@ class EnhanceTest(test_util.ConfigTestCase): with mock.patch('certbot._internal.main.plug_sel.logger.warning') as mock_log: mock_client = self._call(['enhance', '-a', 'webroot', '--redirect']) self.assertTrue(mock_log.called) - self.assertTrue("make sense" in mock_log.call_args[0][0]) + self.assertIn("make sense", mock_log.call_args[0][0]) self.assertTrue(mock_client.enhance_config.called) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @@ -1649,8 +1649,8 @@ class EnhanceTest(test_util.ConfigTestCase): all(getattr(mock_client.config, e) for e in req_enh)) self.assertFalse( any(getattr(mock_client.config, e) for e in not_req_enh)) - self.assertTrue( - "example.com" in mock_client.enhance_config.call_args[0][0]) + self.assertIn( + "example.com", mock_client.enhance_config.call_args[0][0]) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @mock.patch('certbot._internal.main.display_ops.choose_values') @@ -1663,7 +1663,7 @@ class EnhanceTest(test_util.ConfigTestCase): mock_client = self._call(['enhance', '--redirect', '--hsts', '--non-interactive']) self.assertTrue(mock_client.enhance_config.called) - self.assertFalse(mock_choose.called) + self.assertIs(mock_choose.called, False) @mock.patch('certbot._internal.main.display_ops.choose_values') @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins') @@ -1686,7 +1686,7 @@ class EnhanceTest(test_util.ConfigTestCase): mock_pick.return_value = (None, None) mock_pick.side_effect = errors.PluginSelectionError() mock_client = self._call(['enhance', '--hsts']) - self.assertFalse(mock_client.enhance_config.called) + self.assertIs(mock_client.enhance_config.called, False) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @mock.patch('certbot._internal.main.display_ops.choose_values') @@ -1725,7 +1725,7 @@ class InstallTest(test_util.ConfigTestCase): """Tests for certbot._internal.main.install.""" def setUp(self): - super(InstallTest, self).setUp() + super().setUp() self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins') @@ -1870,7 +1870,7 @@ class UpdateAccountTest(test_util.ConfigTestCase): for patch in patches.values(): self.addCleanup(patch.stop) - return super(UpdateAccountTest, self).setUp() + return super().setUp() def _call(self, args): with mock.patch('certbot._internal.main.sys.stdout'), \ diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index fe89fff9f..72a39e2d1 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -68,14 +68,14 @@ class OCSPTestOpenSSL(unittest.TestCase): mock_communicate.communicate.return_value = (None, out.partition("\n")[2]) checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(checker.host_args("x"), ["Host", "x"]) - self.assertEqual(checker.broken, False) + self.assertIs(checker.broken, False) mock_exists.return_value = False mock_popen.call_count = 0 checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 0) self.assertEqual(mock_log.call_count, 1) - self.assertEqual(checker.broken, True) + self.assertIs(checker.broken, True) @mock.patch('certbot.ocsp._determine_ocsp_server') @mock.patch('certbot.ocsp.crypto_util.notAfter') @@ -89,24 +89,24 @@ class OCSPTestOpenSSL(unittest.TestCase): self.checker.broken = True mock_determine.return_value = ("", "") - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.checker.broken = False mock_run.return_value = tuple(openssl_happy[1:]) - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.assertEqual(mock_run.call_count, 0) mock_determine.return_value = ("http://x.co", "x.co") - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) mock_run.side_effect = errors.SubprocessError("Unable to load certificate launcher") - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.assertEqual(mock_run.call_count, 2) # cert expired mock_na.return_value = now mock_determine.return_value = ("", "") count_before = mock_determine.call_count - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.assertEqual(mock_determine.call_count, count_before) def test_determine_ocsp_server(self): @@ -122,22 +122,22 @@ class OCSPTestOpenSSL(unittest.TestCase): # pylint: disable=protected-access mock_run.return_value = openssl_confused from certbot import ocsp - self.assertEqual(ocsp._translate_ocsp_query(*openssl_happy), False) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_confused), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_happy), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_confused), False) self.assertEqual(mock_log.debug.call_count, 1) self.assertEqual(mock_log.warning.call_count, 0) mock_log.debug.call_count = 0 - self.assertEqual(ocsp._translate_ocsp_query(*openssl_unknown), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_unknown), False) self.assertEqual(mock_log.debug.call_count, 1) self.assertEqual(mock_log.warning.call_count, 0) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_expired_ocsp), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_expired_ocsp), False) self.assertEqual(mock_log.debug.call_count, 2) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_broken), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_broken), False) self.assertEqual(mock_log.warning.call_count, 1) mock_log.info.call_count = 0 - self.assertEqual(ocsp._translate_ocsp_query(*openssl_revoked), True) + self.assertIs(ocsp._translate_ocsp_query(*openssl_revoked), True) self.assertEqual(mock_log.info.call_count, 0) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_expired_ocsp_revoked), True) + self.assertIs(ocsp._translate_ocsp_query(*openssl_expired_ocsp_revoked), True) self.assertEqual(mock_log.info.call_count, 1) @@ -236,17 +236,17 @@ class OSCPTestCryptography(unittest.TestCase): with _ocsp_mock(ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, http_status_code=400): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OCSP response in invalid with _ocsp_mock(ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.UNAUTHORIZED): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OCSP response is valid, but certificate status is unknown with _ocsp_mock(ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # The OCSP response says that the certificate is revoked, but certificate # does not contain the OCSP extension. @@ -255,32 +255,32 @@ class OSCPTestCryptography(unittest.TestCase): side_effect=x509.ExtensionNotFound( 'Not found', x509.AuthorityInformationAccessOID.OCSP)): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OCSP response uses an unsupported signature. with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, check_signature_side_effect=UnsupportedAlgorithm('foo')): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OSCP signature response is invalid. with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, check_signature_side_effect=InvalidSignature('foo')): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # Assertion error on OCSP response validity with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, check_signature_side_effect=AssertionError('foo')): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # No responder cert in OCSP response with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) as mocks: mocks['mock_response'].return_value.certificates = [] revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # Responder cert is not signed by certificate issuer with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, @@ -289,7 +289,7 @@ class OSCPTestCryptography(unittest.TestCase): mocks['mock_response'].return_value.certificates[0] = mock.Mock( issuer='fake', subject=cert.subject) revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL): # This mock is necessary to avoid the first call contained in _determine_ocsp_server @@ -300,7 +300,7 @@ class OSCPTestCryptography(unittest.TestCase): side_effect=x509.ExtensionNotFound( 'Not found', x509.AuthorityInformationAccessOID.OCSP)): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) @contextlib.contextmanager diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index bcd190e9b..6eb5dbfce 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -88,7 +88,7 @@ class InstallerTest(test_util.ConfigTestCase): """Tests for certbot.plugins.common.Installer.""" def setUp(self): - super(InstallerTest, self).setUp() + super().setUp() filesystem.mkdir(self.config.config_dir) from certbot.plugins.common import Installer @@ -174,7 +174,7 @@ class InstallerTest(test_util.ConfigTestCase): def test_current_file_hash_in_all_hashes(self): from certbot._internal.constants import ALL_SSL_DHPARAMS_HASHES - self.assertTrue(self._current_ssl_dhparams_hash() in ALL_SSL_DHPARAMS_HASHES, + self.assertIn(self._current_ssl_dhparams_hash(), ALL_SSL_DHPARAMS_HASHES, "Constants.ALL_SSL_DHPARAMS_HASHES must be appended" " with the sha256 hash of self.config.ssl_dhparams when it is updated.") @@ -282,7 +282,7 @@ class InstallVersionControlledFileTest(test_util.TempDirTestCase): """Tests for certbot.plugins.common.install_version_controlled_file.""" def setUp(self): - super(InstallVersionControlledFileTest, self).setUp() + super().setUp() self.hashes = ["someotherhash"] self.dest_path = os.path.join(self.tempdir, "options-ssl-dest.conf") self.hash_path = os.path.join(self.tempdir, ".options-ssl-conf.txt") @@ -330,7 +330,7 @@ class InstallVersionControlledFileTest(test_util.TempDirTestCase): mod_ssl_conf.write("a new line for the wrong hash\n") with mock.patch("certbot.plugins.common.logger") as mock_logger: self._call() - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) self.assertTrue(os.path.isfile(self.dest_path)) self.assertEqual(crypto_util.sha256sum(self.source_path), self._current_file_hash()) @@ -352,7 +352,7 @@ class InstallVersionControlledFileTest(test_util.TempDirTestCase): # only print warning once with mock.patch("certbot.plugins.common.logger") as mock_logger: self._call() - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index 64829c8cc..83dfb41ca 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -2,6 +2,7 @@ import functools import string import unittest +from typing import List try: import mock @@ -74,7 +75,7 @@ class PluginEntryPointTest(unittest.TestCase): name, PluginEntryPoint.entry_point_to_plugin_name(entry_point, with_prefix=True)) def test_description(self): - self.assertTrue("temporary webserver" in self.plugin_ep.description) + self.assertIn("temporary webserver", self.plugin_ep.description) def test_description_with_name(self): self.plugin_ep.plugin_cls = mock.MagicMock(description="Desc") @@ -100,31 +101,31 @@ class PluginEntryPointTest(unittest.TestCase): interfaces.IInstaller, interfaces.IAuthenticator))) def test__init__(self): - self.assertFalse(self.plugin_ep.initialized) - self.assertFalse(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) - self.assertTrue(self.plugin_ep.problem is None) - self.assertTrue(self.plugin_ep.entry_point is EP_SA) + self.assertIs(self.plugin_ep.initialized, False) + self.assertIs(self.plugin_ep.prepared, False) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) + self.assertIsNone(self.plugin_ep.problem) + self.assertIs(self.plugin_ep.entry_point, EP_SA) self.assertEqual("sa", self.plugin_ep.name) - self.assertTrue(self.plugin_ep.plugin_cls is standalone.Authenticator) + self.assertIs(self.plugin_ep.plugin_cls, standalone.Authenticator) def test_init(self): config = mock.MagicMock() plugin = self.plugin_ep.init(config=config) - self.assertTrue(self.plugin_ep.initialized) - self.assertTrue(plugin.config is config) + self.assertIs(self.plugin_ep.initialized, True) + self.assertIs(plugin.config, config) # memoize! - self.assertTrue(self.plugin_ep.init() is plugin) - self.assertTrue(plugin.config is config) + self.assertIs(self.plugin_ep.init(), plugin) + self.assertIs(plugin.config, config) # try to give different config - self.assertTrue(self.plugin_ep.init(123) is plugin) - self.assertTrue(plugin.config is config) + self.assertIs(self.plugin_ep.init(123), plugin) + self.assertIs(plugin.config, config) - self.assertFalse(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) + self.assertIs(self.plugin_ep.prepared, False) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) def test_verify(self): iface1 = mock.MagicMock(__name__="iface1") @@ -154,7 +155,7 @@ class PluginEntryPointTest(unittest.TestCase): self.plugin_ep.init(config=config) self.plugin_ep.prepare() self.assertTrue(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) + self.assertIs(self.plugin_ep.misconfigured, False) # output doesn't matter that much, just test if it runs str(self.plugin_ep) @@ -164,12 +165,11 @@ class PluginEntryPointTest(unittest.TestCase): plugin.prepare.side_effect = errors.MisconfigurationError # pylint: disable=protected-access self.plugin_ep._initialized = plugin - self.assertTrue(isinstance(self.plugin_ep.prepare(), - errors.MisconfigurationError)) + self.assertIsInstance(self.plugin_ep.prepare(), + errors.MisconfigurationError) self.assertTrue(self.plugin_ep.prepared) self.assertTrue(self.plugin_ep.misconfigured) - self.assertTrue(isinstance(self.plugin_ep.problem, - errors.MisconfigurationError)) + self.assertIsInstance(self.plugin_ep.problem, errors.MisconfigurationError) self.assertTrue(self.plugin_ep.available) def test_prepare_no_installation(self): @@ -177,21 +177,20 @@ class PluginEntryPointTest(unittest.TestCase): plugin.prepare.side_effect = errors.NoInstallationError # pylint: disable=protected-access self.plugin_ep._initialized = plugin - self.assertTrue(isinstance(self.plugin_ep.prepare(), - errors.NoInstallationError)) - self.assertTrue(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) + self.assertIsInstance(self.plugin_ep.prepare(), errors.NoInstallationError) + self.assertIs(self.plugin_ep.prepared, True) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) def test_prepare_generic_plugin_error(self): plugin = mock.MagicMock() plugin.prepare.side_effect = errors.PluginError # pylint: disable=protected-access self.plugin_ep._initialized = plugin - self.assertTrue(isinstance(self.plugin_ep.prepare(), errors.PluginError)) + self.assertIsInstance(self.plugin_ep.prepare(), errors.PluginError) self.assertTrue(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) def test_repr(self): self.assertEqual("PluginEntryPoint#sa", repr(self.plugin_ep)) @@ -225,14 +224,14 @@ class PluginsRegistryTest(unittest.TestCase): standalone.Authenticator, webroot.Authenticator, null.Installer, null.Installer] plugins = PluginsRegistry.find_all() - self.assertTrue(plugins["sa"].plugin_cls is standalone.Authenticator) - self.assertTrue(plugins["sa"].entry_point is EP_SA) - self.assertTrue(plugins["wr"].plugin_cls is webroot.Authenticator) - self.assertTrue(plugins["wr"].entry_point is EP_WR) - self.assertTrue(plugins["ep1"].plugin_cls is null.Installer) - self.assertTrue(plugins["ep1"].entry_point is self.ep1) - self.assertTrue(plugins["p1:ep1"].plugin_cls is null.Installer) - self.assertTrue(plugins["p1:ep1"].entry_point is self.ep1) + self.assertIs(plugins["sa"].plugin_cls, standalone.Authenticator) + self.assertIs(plugins["sa"].entry_point, EP_SA) + self.assertIs(plugins["wr"].plugin_cls, webroot.Authenticator) + self.assertIs(plugins["wr"].entry_point, EP_WR) + self.assertIs(plugins["ep1"].plugin_cls, null.Installer) + self.assertIs(plugins["ep1"].entry_point, self.ep1) + self.assertIs(plugins["p1:ep1"].plugin_cls, null.Installer) + self.assertIs(plugins["p1:ep1"].entry_point, self.ep1) def test_getitem(self): self.assertEqual(self.plugin_ep, self.reg["mock"]) @@ -295,10 +294,10 @@ class PluginsRegistryTest(unittest.TestCase): self.assertEqual({}, self.reg.available()._plugins) def test_find_init(self): - self.assertTrue(self.reg.find_init(mock.Mock()) is None) + self.assertIsNone(self.reg.find_init(mock.Mock())) self.plugin_ep.initialized = True - self.assertTrue( - self.reg.find_init(self.plugin_ep.init()) is self.plugin_ep) + self.assertIs( + self.reg.find_init(self.plugin_ep.init()), self.plugin_ep) def test_repr(self): self.plugin_ep.__repr__ = lambda _: "PluginEntryPoint#mock" diff --git a/certbot/tests/plugins/dns_common_lexicon_test.py b/certbot/tests/plugins/dns_common_lexicon_test.py index a67430f3e..40afd107b 100644 --- a/certbot/tests/plugins/dns_common_lexicon_test.py +++ b/certbot/tests/plugins/dns_common_lexicon_test.py @@ -17,7 +17,7 @@ class LexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconCl pass def setUp(self): - super(LexiconClientTest, self).setUp() + super().setUp() self.client = LexiconClientTest._FakeLexiconClient() self.provider_mock = mock.MagicMock() diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index 31761e986..12fac00d0 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -36,7 +36,7 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen fake_file_path = None def setUp(self): - super(DNSAuthenticatorTest, self).setUp() + super().setUp() self.config = DNSAuthenticatorTest._FakeConfig() @@ -164,7 +164,7 @@ class CredentialsConfigurationTest(test_util.TempDirTestCase): class CredentialsConfigurationRequireTest(test_util.TempDirTestCase): def setUp(self): - super(CredentialsConfigurationRequireTest, self).setUp() + super().setUp() self.path = os.path.join(self.tempdir, 'file.ini') @@ -211,20 +211,20 @@ class CredentialsConfigurationRequireTest(test_util.TempDirTestCase): class DomainNameGuessTest(unittest.TestCase): def test_simple_case(self): - self.assertTrue( - 'example.com' in + self.assertIn( + 'example.com', dns_common.base_domain_name_guesses("example.com") ) def test_sub_domain(self): - self.assertTrue( - 'example.com' in + self.assertIn( + 'example.com', dns_common.base_domain_name_guesses("foo.bar.baz.example.com") ) def test_second_level_domain(self): - self.assertTrue( - 'example.co.uk' in + self.assertIn( + 'example.co.uk', dns_common.base_domain_name_guesses("foo.bar.baz.example.co.uk") ) diff --git a/certbot/tests/plugins/enhancements_test.py b/certbot/tests/plugins/enhancements_test.py index a20a6864f..0aa1512b4 100644 --- a/certbot/tests/plugins/enhancements_test.py +++ b/certbot/tests/plugins/enhancements_test.py @@ -15,7 +15,7 @@ class EnhancementTest(test_util.ConfigTestCase): """Tests for new style enhancements in certbot.plugins.enhancements""" def setUp(self): - super(EnhancementTest, self).setUp() + super().setUp() self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 0e552e0c5..c97e08fa6 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -19,7 +19,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): """Tests for certbot._internal.plugins.manual.Authenticator.""" def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() self.http_achall = acme_util.HTTP01_A self.dns_achall = acme_util.DNS01_A self.dns_achall_2 = acme_util.DNS01_A_2 @@ -52,7 +52,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.assertRaises(errors.HookCommandNotFound, self.auth.prepare) def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), str)) + self.assertIsInstance(self.auth.more_info(), str) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref('example.org'), @@ -96,9 +96,8 @@ class AuthenticatorTest(test_util.TempDirTestCase): [achall.response(achall.account_key) for achall in self.achalls]) for i, (args, kwargs) in enumerate(mock_get_utility().notification.call_args_list): achall = self.achalls[i] - self.assertTrue( - achall.validation(achall.account_key) in args[0]) - self.assertFalse(kwargs['wrap']) + self.assertIn(achall.validation(achall.account_key), args[0]) + self.assertIs(kwargs['wrap'], False) def test_cleanup(self): self.config.manual_auth_hook = ('{0} -c "import sys; sys.stdout.write(\'foo\')"' @@ -119,7 +118,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): os.environ['CERTBOT_TOKEN'], achall.chall.encode('token')) else: - self.assertFalse('CERTBOT_TOKEN' in os.environ) + self.assertNotIn('CERTBOT_TOKEN', os.environ) if __name__ == '__main__': diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index dad5b270a..dfdd0a7de 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -15,7 +15,7 @@ class InstallerTest(unittest.TestCase): self.installer = Installer(config=mock.MagicMock(), name="null") def test_it(self): - self.assertTrue(isinstance(self.installer.more_info(), str)) + self.assertIsInstance(self.installer.more_info(), str) self.assertEqual([], self.installer.get_all_names()) self.assertEqual([], self.installer.supported_enhancements()) diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index 027ebdc35..60917626a 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -1,6 +1,7 @@ """Tests for letsencrypt.plugins.selection""" import sys import unittest +from typing import List try: import mock @@ -69,7 +70,7 @@ class PickPluginTest(unittest.TestCase): self.assertEqual(1, self.reg.visible().ifaces.call_count) def test_no_candidate(self): - self.assertTrue(self._call() is None) + self.assertIsNone(self._call()) def test_single(self): plugin_ep = mock.MagicMock() @@ -87,7 +88,7 @@ class PickPluginTest(unittest.TestCase): self.reg.visible().ifaces().verify().available.return_value = { "bar": plugin_ep} - self.assertTrue(self._call() is None) + self.assertIsNone(self._call()) def test_multiple(self): plugin_ep = mock.MagicMock() @@ -110,7 +111,7 @@ class PickPluginTest(unittest.TestCase): with mock.patch("certbot._internal.plugins.selection.choose_plugin") as mock_choose: mock_choose.return_value = None - self.assertTrue(self._call() is None) + self.assertIsNone(self._call()) class ChoosePluginTest(unittest.TestCase): @@ -152,34 +153,14 @@ class ChoosePluginTest(unittest.TestCase): @test_util.patch_get_utility("certbot._internal.plugins.selection.z_util") def test_no_choice(self, mock_util): mock_util().menu.return_value = (display_util.CANCEL, 0) - self.assertTrue(self._call() is None) - - @test_util.patch_get_utility("certbot._internal.plugins.selection.z_util") - def test_new_interaction_avoidance(self, mock_util): - mock_nginx = mock.Mock( - description_with_name="n", misconfigured=False) - mock_nginx.init().more_info.return_value = "nginx plugin" - mock_nginx.name = "nginx" - self.plugins[1] = mock_nginx - mock_util().menu.return_value = (display_util.CANCEL, 0) - - unset_cb_auto = os.environ.get("CERTBOT_AUTO") is None - if unset_cb_auto: - os.environ["CERTBOT_AUTO"] = "foo" - try: - self._call() - finally: - if unset_cb_auto: - del os.environ["CERTBOT_AUTO"] - - self.assertTrue("default" in mock_util().menu.call_args[1]) + self.assertIsNone(self._call()) class GetUnpreparedInstallerTest(test_util.ConfigTestCase): """Tests for certbot._internal.plugins.selection.get_unprepared_installer.""" def setUp(self): - super(GetUnpreparedInstallerTest, self).setUp() + super().setUp() self.mock_apache_fail_ep = mock.Mock( description_with_name="afail") self.mock_apache_fail_ep.check_name = lambda name: name == "afail" @@ -199,7 +180,7 @@ class GetUnpreparedInstallerTest(test_util.ConfigTestCase): def test_no_installer_defined(self): self.config.configurator = None - self.assertEqual(self._call(), None) + self.assertIsNone(self._call()) def test_no_available_installers(self): self.config.configurator = "apache" @@ -209,7 +190,7 @@ class GetUnpreparedInstallerTest(test_util.ConfigTestCase): def test_get_plugin(self): self.config.configurator = "apache" installer = self._call() - self.assertTrue(installer is self.mock_apache_plugin) + self.assertIs(installer, self.mock_apache_plugin) def test_multiple_installers_returned(self): self.config.configurator = "apache" diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 80347d35a..6f2ae91ba 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -1,8 +1,8 @@ """Tests for certbot._internal.plugins.standalone.""" -# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi +import errno import socket -from socket import errno as socket_errors # type: ignore import unittest +from typing import Dict, Set, Tuple import josepy as jose try: @@ -29,9 +29,8 @@ class ServerManagerTest(unittest.TestCase): self.mgr = ServerManager(self.certs, self.http_01_resources) def test_init(self): - self.assertTrue(self.mgr.certs is self.certs) - self.assertTrue( - self.mgr.http_01_resources is self.http_01_resources) + self.assertIs(self.mgr.certs, self.certs) + self.assertIs(self.mgr.http_01_resources, self.http_01_resources) def _test_run_stop(self, challenge_type): server = self.mgr.run(port=0, challenge_type=challenge_type) @@ -48,7 +47,7 @@ class ServerManagerTest(unittest.TestCase): port = server.getsocknames()[0][1] server2 = self.mgr.run(port=port, challenge_type=challenges.HTTP01) self.assertEqual(self.mgr.running(), {port: server}) - self.assertTrue(server is server2) + self.assertIs(server, server2) self.mgr.stop(port) self.assertEqual(self.mgr.running(), {}) @@ -89,7 +88,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.servers = mock.MagicMock() def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), str)) + self.assertIsInstance(self.auth.more_info(), str) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref(domain=None), @@ -105,8 +104,8 @@ class AuthenticatorTest(unittest.TestCase): @test_util.patch_get_utility() def test_perform_eaddrinuse_retry(self, mock_get_utility): mock_utility = mock_get_utility() - errno = socket_errors.EADDRINUSE - error = errors.StandaloneBindError(mock.MagicMock(errno=errno), -1) + encountered_errno = errno.EADDRINUSE + error = errors.StandaloneBindError(mock.MagicMock(errno=encountered_errno), -1) self.auth.servers.run.side_effect = [error] + 2 * [mock.MagicMock()] mock_yesno = mock_utility.yesno mock_yesno.return_value = True @@ -120,26 +119,26 @@ class AuthenticatorTest(unittest.TestCase): mock_yesno = mock_utility.yesno mock_yesno.return_value = False - errno = socket_errors.EADDRINUSE - self.assertRaises(errors.PluginError, self._fail_perform, errno) + encountered_errno = errno.EADDRINUSE + self.assertRaises(errors.PluginError, self._fail_perform, encountered_errno) self._assert_correct_yesno_call(mock_yesno) def _assert_correct_yesno_call(self, mock_yesno): yesno_args, yesno_kwargs = mock_yesno.call_args - self.assertTrue("in use" in yesno_args[0]) + self.assertIn("in use", yesno_args[0]) self.assertFalse(yesno_kwargs.get("default", True)) def test_perform_eacces(self): - errno = socket_errors.EACCES - self.assertRaises(errors.PluginError, self._fail_perform, errno) + encountered_errno = errno.EACCES + self.assertRaises(errors.PluginError, self._fail_perform, encountered_errno) def test_perform_unexpected_socket_error(self): - errno = socket_errors.ENOTCONN + encountered_errno = errno.ENOTCONN self.assertRaises( - errors.StandaloneBindError, self._fail_perform, errno) + errors.StandaloneBindError, self._fail_perform, encountered_errno) - def _fail_perform(self, errno): - error = errors.StandaloneBindError(mock.MagicMock(errno=errno), -1) + def _fail_perform(self, encountered_errno): + error = errors.StandaloneBindError(mock.MagicMock(errno=encountered_errno), -1) self.auth.servers.run.side_effect = error self.auth.perform(self._get_achalls()) diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index 2999d306e..d01845510 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -18,7 +18,7 @@ class PluginStorageTest(test_util.ConfigTestCase): """Test for certbot.plugins.storage.PluginStorage""" def setUp(self): - super(PluginStorageTest, self).setUp() + super().setUp() self.plugin_cls = common.Installer filesystem.mkdir(self.config.config_dir) with mock.patch("certbot.reverter.util"): @@ -31,7 +31,7 @@ class PluginStorageTest(test_util.ConfigTestCase): # When unable to read file that exists mock_open = mock.mock_open() mock_open.side_effect = IOError - self.plugin.storage.storagepath = os.path.join(self.config.config_dir, + self.plugin.storage._storagepath = os.path.join(self.config.config_dir, ".pluginstorage.json") with mock.patch("builtins.open", mock_open): with mock.patch('certbot.compat.os.path.isfile', return_value=True): @@ -49,7 +49,7 @@ class PluginStorageTest(test_util.ConfigTestCase): self.assertRaises(KeyError, nocontent.storage.fetch, "value") self.assertTrue(mock_log.called) - self.assertTrue("no values loaded" in mock_log.call_args[0][0]) + self.assertIn("no values loaded", mock_log.call_args[0][0]) def test_load_errors_corrupted(self): with open(os.path.join(self.config.config_dir, @@ -61,17 +61,17 @@ class PluginStorageTest(test_util.ConfigTestCase): self.assertRaises(errors.PluginError, corrupted.storage.fetch, "value") - self.assertTrue("is corrupted" in mock_log.call_args[0][0]) + self.assertIn("is corrupted", mock_log.call_args[0][0]) def test_save_errors_cant_serialize(self): with mock.patch("certbot.plugins.storage.logger.error") as mock_log: # Set data as something that can't be serialized self.plugin.storage._initialized = True # pylint: disable=protected-access - self.plugin.storage.storagepath = "/tmp/whatever" + self.plugin.storage._storagepath = "/tmp/whatever" self.plugin.storage._data = self.plugin_cls # pylint: disable=protected-access self.assertRaises(errors.PluginStorageError, self.plugin.storage.save) - self.assertTrue("Could not serialize" in mock_log.call_args[0][0]) + self.assertIn("Could not serialize", mock_log.call_args[0][0]) def test_save_errors_unable_to_write_file(self): mock_open = mock.mock_open() @@ -80,9 +80,10 @@ class PluginStorageTest(test_util.ConfigTestCase): with mock.patch("certbot.plugins.storage.logger.error") as mock_log: self.plugin.storage._data = {"valid": "data"} # pylint: disable=protected-access self.plugin.storage._initialized = True # pylint: disable=protected-access + self.plugin.storage._storagepath = "/tmp/whatever" self.assertRaises(errors.PluginStorageError, self.plugin.storage.save) - self.assertTrue("Could not write" in mock_log.call_args[0][0]) + self.assertIn("Could not write", mock_log.call_args[0][0]) def test_save_uninitialized(self): with mock.patch("certbot.reverter.util"): @@ -113,7 +114,7 @@ class PluginStorageTest(test_util.ConfigTestCase): ".pluginstorage.json"), 'r') as fh: psdata = fh.read() psjson = json.loads(psdata) - self.assertTrue("mockplugin" in psjson.keys()) + self.assertIn("mockplugin", psjson.keys()) self.assertEqual(len(psjson), 1) self.assertEqual(psjson["mockplugin"]["testkey"], "testvalue") diff --git a/certbot/tests/plugins/util_test.py b/certbot/tests/plugins/util_test.py index 9387b4ae7..1b4fcd652 100644 --- a/certbot/tests/plugins/util_test.py +++ b/certbot/tests/plugins/util_test.py @@ -30,7 +30,7 @@ class PathSurgeryTest(unittest.TestCase): with mock.patch.dict('os.environ', all_path): with mock.patch('certbot.util.exe_exists') as mock_exists: mock_exists.return_value = True - self.assertEqual(path_surgery("eg"), True) + self.assertIs(path_surgery("eg"), True) self.assertEqual(mock_debug.call_count, 0) self.assertEqual(os.environ["PATH"], all_path["PATH"]) if os.name != 'nt': @@ -39,9 +39,9 @@ class PathSurgeryTest(unittest.TestCase): with mock.patch.dict('os.environ', no_path): path_surgery("thingy") self.assertEqual(mock_debug.call_count, 2 if os.name != 'nt' else 1) - self.assertTrue("Failed to find" in mock_debug.call_args[0][0]) - self.assertTrue("/usr/local/bin" in os.environ["PATH"]) - self.assertTrue("/tmp" in os.environ["PATH"]) + self.assertIn("Failed to find", mock_debug.call_args[0][0]) + self.assertIn("/usr/local/bin", os.environ["PATH"]) + self.assertIn("/tmp", os.environ["PATH"]) if __name__ == "__main__": diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index e6fbd8e88..f158486b6 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -58,8 +58,8 @@ class AuthenticatorTest(unittest.TestCase): def test_more_info(self): more_info = self.auth.more_info() - self.assertTrue(isinstance(more_info, str)) - self.assertTrue(self.path in more_info) + self.assertIsInstance(more_info, str) + self.assertIn(self.path, more_info) def test_add_parser_arguments(self): add = mock.MagicMock() @@ -79,7 +79,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.perform([self.achall]) self.assertTrue(mock_display.menu.called) for call in mock_display.menu.call_args_list: - self.assertTrue(self.achall.domain in call[0][0]) + self.assertIn(self.achall.domain, call[0][0]) self.assertTrue(all( webroot in call[0][1] for webroot in self.config.webroot_map.values())) @@ -96,7 +96,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertRaises(errors.PluginError, self.auth.perform, [self.achall]) self.assertTrue(mock_display.menu.called) for call in mock_display.menu.call_args_list: - self.assertTrue(self.achall.domain in call[0][0]) + self.assertIn(self.achall.domain, call[0][0]) self.assertTrue(all( webroot in call[0][1] for webroot in self.config.webroot_map.values())) @@ -141,7 +141,8 @@ class AuthenticatorTest(unittest.TestCase): f.write("thingimy") filesystem.chmod(self.path, 0o000) try: - open(permission_canary, "r") + with open(permission_canary, "r"): + pass print("Warning, running tests as root skips permissions tests...") except IOError: # ok, permissions work, test away... diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index 7c9c53fb4..edee8df6c 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -115,7 +115,7 @@ class RenewalTest(test_util.ConfigTestCase): renewal_candidate = renewal._reconstitute(lineage_config, rc_path) # This means that manual_public_ip_logging_ok was not modified in the config based on its # value in the renewal conf file - self.assertTrue(isinstance(lineage_config.manual_public_ip_logging_ok, mock.MagicMock)) + self.assertIsInstance(lineage_config.manual_public_ip_logging_ok, mock.MagicMock) class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): @@ -129,7 +129,7 @@ class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): def test_allow_subset_of_names_success(self, mock_set_by_cli): mock_set_by_cli.return_value = False self._call(self.config, {'allow_subset_of_names': 'True'}) - self.assertTrue(self.config.allow_subset_of_names is True) + self.assertIs(self.config.allow_subset_of_names, True) @mock.patch('certbot._internal.renewal.cli.set_by_cli') def test_allow_subset_of_names_failure(self, mock_set_by_cli): @@ -164,7 +164,7 @@ class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): def test_must_staple_success(self, mock_set_by_cli): mock_set_by_cli.return_value = False self._call(self.config, {'must_staple': 'True'}) - self.assertTrue(self.config.must_staple is True) + self.assertIs(self.config.must_staple, True) @mock.patch('certbot._internal.renewal.cli.set_by_cli') def test_must_staple_failure(self, mock_set_by_cli): diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index b5ecddb5a..55faff2a4 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -17,7 +17,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): """Tests for interfaces.RenewDeployer and interfaces.GenericUpdater""" def setUp(self): - super(RenewUpdaterTest, self).setUp() + super().setUp() self.generic_updater = mock.MagicMock(spec=interfaces.GenericUpdater) self.generic_updater.restart = mock.MagicMock() self.renew_deployer = mock.MagicMock(spec=interfaces.RenewDeployer) @@ -42,7 +42,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): mock_generic_updater.generic_updates.reset_mock() updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertEqual(mock_generic_updater.generic_updates.call_count, 1) - self.assertFalse(mock_generic_updater.restart.called) + self.assertIs(mock_generic_updater.restart.called, False) def test_renew_deployer(self): lineage = mock.MagicMock() @@ -83,13 +83,13 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.config.disable_renew_updates = True mock_geti.return_value = self.mockinstaller updater.run_generic_updaters(self.config, mock.MagicMock(), None) - self.assertFalse(self.mockinstaller.update_autohsts.called) + self.assertIs(self.mockinstaller.update_autohsts.called, False) def test_enhancement_deployer_not_called(self): self.config.disable_renew_updates = True updater.run_renewal_deployer(self.config, mock.MagicMock(), self.mockinstaller) - self.assertFalse(self.mockinstaller.deploy_autohsts.called) + self.assertIs(self.mockinstaller.deploy_autohsts.called, False) @mock.patch('certbot._internal.plugins.selection.get_unprepared_installer') def test_enhancement_no_updater(self, mock_geti): @@ -105,7 +105,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): mock_geti.return_value = self.mockinstaller with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): updater.run_generic_updaters(self.config, mock.MagicMock(), None) - self.assertFalse(self.mockinstaller.update_autohsts.called) + self.assertIs(self.mockinstaller.update_autohsts.called, False) def test_enhancement_no_deployer(self): FAKEINDEX = [ @@ -120,7 +120,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): updater.run_renewal_deployer(self.config, mock.MagicMock(), self.mockinstaller) - self.assertFalse(self.mockinstaller.deploy_autohsts.called) + self.assertIs(self.mockinstaller.deploy_autohsts.called, False) if __name__ == '__main__': diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 7a37f782e..c4af40591 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -26,8 +26,8 @@ class ReporterTest(unittest.TestCase): self.reporter.add_message("Line 1\nLine 2", self.reporter.LOW_PRIORITY) self.reporter.print_messages() output = sys.stdout.getvalue() # type: ignore - self.assertTrue("Line 1\n" in output) - self.assertTrue("Line 2" in output) + self.assertIn("Line 1\n", output) + self.assertIn("Line 2", output) def test_tty_print_empty(self): sys.stdout.isatty = lambda: True # type: ignore @@ -60,10 +60,10 @@ class ReporterTest(unittest.TestCase): self._add_messages() self.reporter.print_messages() output = sys.stdout.getvalue() # type: ignore - self.assertTrue("IMPORTANT NOTES:" in output) - self.assertTrue("High" in output) - self.assertTrue("Med" in output) - self.assertTrue("Low" in output) + self.assertIn("IMPORTANT NOTES:", output) + self.assertIn("High", output) + self.assertIn("Med", output) + self.assertIn("Low", output) def _unsuccessful_exit_common(self): self._add_messages() @@ -72,10 +72,10 @@ class ReporterTest(unittest.TestCase): except ValueError: self.reporter.print_messages() output = sys.stdout.getvalue() # type: ignore - self.assertTrue("IMPORTANT NOTES:" in output) - self.assertTrue("High" in output) - self.assertTrue("Med" not in output) - self.assertTrue("Low" not in output) + self.assertIn("IMPORTANT NOTES:", output) + self.assertIn("High", output) + self.assertNotIn("Med", output) + self.assertNotIn("Low", output) def _add_messages(self): self.reporter.add_message("High", self.reporter.HIGH_PRIORITY) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index af01a9a1b..e8d85d4d1 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -18,7 +18,7 @@ from certbot.tests import util as test_util class ReverterCheckpointLocalTest(test_util.ConfigTestCase): """Test the Reverter Class.""" def setUp(self): - super(ReverterCheckpointLocalTest, self).setUp() + super().setUp() from certbot.reverter import Reverter # Disable spurious errors... we are trying to test for them @@ -48,7 +48,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): no_change = os.path.join(self.reverter.config.backup_dir, path, "CHANGES_SINCE") with open(no_change, "r") as f: x = f.read() - self.assertTrue("No changes" in x) + self.assertIn("No changes", x) def test_basic_add_to_temp_checkpoint(self): # These shouldn't conflict even though they are both named config.txt @@ -280,7 +280,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): class TestFullCheckpointsReverter(test_util.ConfigTestCase): """Tests functions having to deal with full checkpoints.""" def setUp(self): - super(TestFullCheckpointsReverter, self).setUp() + super().setUp() from certbot.reverter import Reverter # Disable spurious errors... logging.disable(logging.CRITICAL) @@ -328,8 +328,8 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): # One dir left... check title all_dirs = os.listdir(self.config.backup_dir) self.assertEqual(len(all_dirs), 1) - self.assertTrue( - "First Checkpoint" in get_save_notes( + self.assertIn( + "First Checkpoint", get_save_notes( os.path.join(self.config.backup_dir, all_dirs[0]))) # Final rollback self.reverter.rollback_checkpoints(1) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 1e3a1ffd8..aa5910f1e 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -99,7 +99,7 @@ class BaseRenewableCertTest(test_util.ConfigTestCase): def setUp(self): from certbot._internal import storage - super(BaseRenewableCertTest, self).setUp() + super().setUp() # TODO: maybe provide NamespaceConfig.make_dirs? # TODO: main() should create those dirs, c.f. #902 @@ -195,11 +195,11 @@ class RenewableCertTests(BaseRenewableCertTest): from certbot._internal import storage self._write_out_ex_kinds() - self.assertTrue("version" not in self.config_file) + self.assertNotIn("version", self.config_file) with mock.patch("certbot._internal.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) def test_renewal_newer_version(self): from certbot._internal import storage @@ -211,7 +211,7 @@ class RenewableCertTests(BaseRenewableCertTest): with mock.patch("certbot._internal.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) self.assertTrue(mock_logger.info.called) - self.assertTrue("version" in mock_logger.info.call_args[0][0]) + self.assertIn("version", mock_logger.info.call_args[0][0]) def test_consistent(self): # pylint: disable=protected-access @@ -282,7 +282,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version("cert"), 10) def test_no_current_version(self): - self.assertEqual(self.test_rc.current_version("cert"), None) + self.assertIsNone(self.test_rc.current_version("cert")) def test_latest_and_next_versions(self): for ver in range(1, 6): @@ -314,12 +314,12 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.latest_common_version = mock.Mock() mock_has_pending.return_value = False - self.assertEqual(self.test_rc.ensure_deployed(), True) + self.assertIs(self.test_rc.ensure_deployed(), True) self.assertEqual(mock_update.call_count, 0) self.assertEqual(mock_logger.warning.call_count, 0) mock_has_pending.return_value = True - self.assertEqual(self.test_rc.ensure_deployed(), False) + self.assertIs(self.test_rc.ensure_deployed(), False) self.assertEqual(mock_update.call_count, 1) self.assertEqual(mock_logger.warning.call_count, 1) @@ -586,7 +586,7 @@ class RenewableCertTests(BaseRenewableCertTest): self._write_out_kind(kind, 1) self.test_rc.update_all_links_to(1) self.test_rc.save_successor(1, b"newcert", None, b"new chain", self.config) - self.assertFalse(mock_ownership.called) + self.assertIs(mock_ownership.called, False) self.test_rc.save_successor(2, b"newcert", b"new_privkey", b"new chain", self.config) self.assertTrue(mock_ownership.called) @@ -767,7 +767,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_server(self): self.test_rc.configuration["renewalparams"] = {} - self.assertEqual(self.test_rc.server, None) + self.assertIsNone(self.test_rc.server) rp = self.test_rc.configuration["renewalparams"] rp["server"] = "https://acme.example/dir" self.assertEqual(self.test_rc.server, "https://acme.example/dir") @@ -775,15 +775,15 @@ class RenewableCertTests(BaseRenewableCertTest): def test_is_test_cert(self): self.test_rc.configuration["renewalparams"] = {} rp = self.test_rc.configuration["renewalparams"] - self.assertEqual(self.test_rc.is_test_cert, False) + self.assertIs(self.test_rc.is_test_cert, False) rp["server"] = "https://acme-staging-v02.api.letsencrypt.org/directory" - self.assertEqual(self.test_rc.is_test_cert, True) + self.assertIs(self.test_rc.is_test_cert, True) rp["server"] = "https://staging.someotherca.com/directory" - self.assertEqual(self.test_rc.is_test_cert, True) + self.assertIs(self.test_rc.is_test_cert, True) rp["server"] = "https://acme-v01.api.letsencrypt.org/directory" - self.assertEqual(self.test_rc.is_test_cert, False) + self.assertIs(self.test_rc.is_test_cert, False) rp["server"] = "https://acme-v02.api.letsencrypt.org/directory" - self.assertEqual(self.test_rc.is_test_cert, False) + self.assertIs(self.test_rc.is_test_cert, False) def test_missing_cert(self): from certbot._internal import storage @@ -817,13 +817,13 @@ class RenewableCertTests(BaseRenewableCertTest): with open(temp2, "r") as f: content = f.read() # useful value was updated - self.assertTrue("useful = new_value" in content) + self.assertIn("useful = new_value", content) # associated comment was preserved - self.assertTrue("A useful value" in content) + self.assertIn("A useful value", content) # useless value was deleted - self.assertTrue("useless" not in content) + self.assertNotIn("useless", content) # check version was stored - self.assertTrue("version = {0}".format(certbot.__version__) in content) + self.assertIn("version = {0}".format(certbot.__version__), content) # ensure permissions are copied self.assertEqual(stat.S_IMODE(os.lstat(temp).st_mode), stat.S_IMODE(os.lstat(temp2).st_mode)) @@ -846,7 +846,7 @@ class RenewableCertTests(BaseRenewableCertTest): class DeleteFilesTest(BaseRenewableCertTest): """Tests for certbot._internal.storage.delete_files""" def setUp(self): - super(DeleteFilesTest, self).setUp() + super().setUp() for kind in ALL_FOUR: kind_path = os.path.join(self.config.config_dir, "live", "example.org", @@ -936,7 +936,7 @@ class DeleteFilesTest(BaseRenewableCertTest): class CertPathForCertNameTest(BaseRenewableCertTest): """Test for certbot._internal.storage.cert_path_for_cert_name""" def setUp(self): - super(CertPathForCertNameTest, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 18947c342..a78b614cf 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -108,7 +108,7 @@ class LockDirUntilExit(test_util.TempDirTestCase): return lock_dir_until_exit(*args, **kwargs) def setUp(self): - super(LockDirUntilExit, self).setUp() + super().setUp() # reset global state from other tests import certbot.util reload_module(certbot.util) @@ -164,7 +164,7 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase): """ def setUp(self): - super(MakeOrVerifyDirTest, self).setUp() + super().setUp() self.path = os.path.join(self.tempdir, "foo") filesystem.mkdir(self.path, 0o600) @@ -196,7 +196,7 @@ class UniqueFileTest(test_util.TempDirTestCase): """Tests for certbot.util.unique_file.""" def setUp(self): - super(UniqueFileTest, self).setUp() + super().setUp() self.default_name = os.path.join(self.tempdir, "foo.txt") @@ -260,7 +260,7 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): def test_basic(self): f, path = self._call("wow") - self.assertTrue(isinstance(f, file_type)) + self.assertIsInstance(f, file_type) self.assertEqual(os.path.join(self.tempdir, "wow.conf"), path) f.close() @@ -269,9 +269,9 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): for _ in range(10): items.append(self._call("wow")) f, name = items[-1] - self.assertTrue(isinstance(f, file_type)) - self.assertTrue(isinstance(name, str)) - self.assertTrue("wow-0009.conf" in name) + self.assertIsInstance(f, file_type) + self.assertIsInstance(name, str) + self.assertIn("wow-0009.conf", name) for f, _ in items: f.close() @@ -284,7 +284,7 @@ class SafelyRemoveTest(test_util.TempDirTestCase): """Tests for certbot.util.safely_remove.""" def setUp(self): - super(SafelyRemoveTest, self).setUp() + super().setUp() self.path = os.path.join(self.tempdir, "foo") @@ -349,7 +349,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): with mock.patch("certbot.util.logger.warning") as mock_warn: self.parser.parse_args(["--old-option"]) self.assertEqual(mock_warn.call_count, 1) - self.assertTrue("is deprecated" in mock_warn.call_args[0][0]) + self.assertIn("is deprecated", mock_warn.call_args[0][0]) self.assertEqual("--old-option", mock_warn.call_args[0][1]) def test_warning_with_arg(self): @@ -357,7 +357,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): with mock.patch("certbot.util.logger.warning") as mock_warn: self.parser.parse_args(["--old-option", "42"]) self.assertEqual(mock_warn.call_count, 1) - self.assertTrue("is deprecated" in mock_warn.call_args[0][0]) + self.assertIn("is deprecated", mock_warn.call_args[0][0]) self.assertEqual("--old-option", mock_warn.call_args[0][1]) def test_help(self): @@ -368,7 +368,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): self.parser.parse_args(["-h"]) except SystemExit: pass - self.assertTrue("--old-option" not in stdout.getvalue()) + self.assertNotIn("--old-option", stdout.getvalue()) def test_set_constant(self): """Test when ACTION_TYPES_THAT_DONT_NEED_A_VALUE is a set. @@ -519,7 +519,7 @@ class OsInfoTest(unittest.TestCase): m_distro.like.return_value = "first debian third" id_likes = cbutil.get_systemd_os_like() self.assertEqual(len(id_likes), 3) - self.assertTrue("debian" in id_likes) + self.assertIn("debian", id_likes) @mock.patch("certbot.util.distro") @unittest.skipUnless(sys.platform.startswith("linux"), "requires Linux") @@ -610,7 +610,7 @@ class AtexitRegisterTest(unittest.TestCase): def test_not_called(self): self._test_common(initial_pid=-1) - self.assertFalse(self.func.called) + self.assertIs(self.func.called, False) def _test_common(self, initial_pid): with mock.patch('certbot.util._INITIAL_PID', initial_pid): diff --git a/letsencrypt-auto b/letsencrypt-auto index 24af0b4c3..c37c45596 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.13.0" +LE_AUTO_VERSION="1.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -800,6 +800,7 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 @@ -1488,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.13.0 \ - --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ - --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 -acme==1.13.0 \ - --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ - --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 -certbot-apache==1.13.0 \ - --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ - --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 -certbot-nginx==1.13.0 \ - --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ - --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 +certbot==1.14.0 \ + --hash=sha256:67b4d26ceaea6c7f8325d0d45169e7a165a2cabc7122c84bc971ba068ca19cca \ + --hash=sha256:959ea90c6bb8dca38eab9772722cb940972ef6afcd5f15deef08b3c3636841eb +acme==1.14.0 \ + --hash=sha256:4f48c41261202f1a389ec2986b2580b58f53e0d5a1ae2463b34318d78b87fc66 \ + --hash=sha256:61daccfb0343628cbbca551a7fc4c82482113952c21db3fe0c585b7c98fa1c35 +certbot-apache==1.14.0 \ + --hash=sha256:b757038db23db707c44630fecb46e99172bd791f0db5a8e623c0842613c4d3d9 \ + --hash=sha256:887fe4a21af2de1e5c2c9428bacba6eb7c1219257bc70f1a1d8447c8a321adb0 +certbot-nginx==1.14.0 \ + --hash=sha256:8916a815437988d6c192df9f035bb7a176eab20eee0956677b335d0698d243fb \ + --hash=sha256:cc2a8a0de56d9bb6b2efbda6c80c647dad8db2bb90675cac03ade94bd5fc8597 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/build.py b/letsencrypt-auto-source/build.py deleted file mode 100755 index a1e40fe44..000000000 --- a/letsencrypt-auto-source/build.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -"""Stitch together the letsencrypt-auto script. - -Implement a simple templating language in which {{ some/file }} turns into the -contents of the file at ./pieces/some/file except for certain tokens which have -other, special definitions. - -""" -from os.path import abspath, dirname, join -import re - -from version import certbot_version, file_contents - - -DIR = dirname(abspath(__file__)) - - -def build(version=None, requirements=None): - """Return the built contents of the letsencrypt-auto script. - - :arg version: The version to attach to the script. Default: the version of - the certbot package - :arg requirements: The contents of the requirements file to embed. Default: - contents of dependency-requirements.txt, letsencrypt-requirements.txt, - and certbot-requirements.txt - - """ - special_replacements = { - 'LE_AUTO_VERSION': version or certbot_version(DIR) - } - if requirements: - special_replacements['dependency-requirements.txt'] = '' - special_replacements['letsencrypt-requirements.txt'] = '' - special_replacements['certbot-requirements.txt'] = requirements - - def replacer(match): - token = match.group(1) - if token in special_replacements: - return special_replacements[token] - else: - return file_contents(join(DIR, 'pieces', token)) - - return re.sub(r'{{\s*([A-Za-z0-9_./-]+)\s*}}', - replacer, - file_contents(join(DIR, 'letsencrypt-auto.template'))) - - -def main(): - with open(join(DIR, 'letsencrypt-auto'), 'w') as out: - out.write(build()) - - -if __name__ == '__main__': - main() diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index fbd16f2b0..c0cf63418 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmA+sv4ACgkQTRfJlc2X -dfLG+ggAvwUCqy06UZd8jZhOZFUoi8nWHG2TMnm/4CW4G9PPCjsCoQplhaAaUCrR -PAv0vtsG1rBKWekICg6IBTzLVioH9xRPUkpfbVQhT1c1T/5CqMsFFXR5p9YAKRe7 -hlOb7VRN10bdCS4JThRPNhdWFdWKZXYKcIOObWA/FX2GacxMHuLwPpSsbt2NRffy -qz1ZOWxvr289aWEbZWfyBiI2bxQ7wlMEbZ/JLUXDe46ETDxzENu+c0x2109ha6m4 -wHmUS0r16ps/n9DueTZGJf3C26mU+cIB+LgNvOcibBo+0Ly7t+OiBHYbkFXS2KTQ -RbH4bPZrsduUOzhE8wIsSUIsVGDleQ== -=jYnL +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmBsmUkACgkQTRfJlc2X +dfI7Bwf9FkNrf1HEh2G3uk1p+qLMd/s5kcVV2udK2FkRELee5nHlLZx2YmHA/8ID +gqsk8EsyRZNMX374nGrPm0syykdEsyVtMJTbHCEr+Ms3l54ZgE3HV6ywnhWSlAFo +Za50kdzhodBVTS5AEADbCKLKObVAWwO3fFKtKyv/iY29ykpHK0KSHCKRII3iQU7l +dnR6u35Z0wgfEmDxsH27K6uo0YepZaEL70qHHFk93MhCh9Z15rO17gRpsVzz7Z1j +YClI6h2K/VOfZtbkoQvoks7s+xd75Kjr3GNH+cznkJx8gNWSZLfkc1XX4Bjdm4GG +IWz3Ezy8tFg6PtITb7y+aIg75kWx4w== +=zEy4 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index b0b25759f..c37c45596 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.14.0.dev0" +LE_AUTO_VERSION="1.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1489,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.13.0 \ - --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ - --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 -acme==1.13.0 \ - --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ - --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 -certbot-apache==1.13.0 \ - --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ - --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 -certbot-nginx==1.13.0 \ - --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ - --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 +certbot==1.14.0 \ + --hash=sha256:67b4d26ceaea6c7f8325d0d45169e7a165a2cabc7122c84bc971ba068ca19cca \ + --hash=sha256:959ea90c6bb8dca38eab9772722cb940972ef6afcd5f15deef08b3c3636841eb +acme==1.14.0 \ + --hash=sha256:4f48c41261202f1a389ec2986b2580b58f53e0d5a1ae2463b34318d78b87fc66 \ + --hash=sha256:61daccfb0343628cbbca551a7fc4c82482113952c21db3fe0c585b7c98fa1c35 +certbot-apache==1.14.0 \ + --hash=sha256:b757038db23db707c44630fecb46e99172bd791f0db5a8e623c0842613c4d3d9 \ + --hash=sha256:887fe4a21af2de1e5c2c9428bacba6eb7c1219257bc70f1a1d8447c8a321adb0 +certbot-nginx==1.14.0 \ + --hash=sha256:8916a815437988d6c192df9f035bb7a176eab20eee0956677b335d0698d243fb \ + --hash=sha256:cc2a8a0de56d9bb6b2efbda6c80c647dad8db2bb90675cac03ade94bd5fc8597 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 3516f77b0..b297bdfb3 100644 --- a/letsencrypt-auto-source/letsencrypt-auto.sig +++ b/letsencrypt-auto-source/letsencrypt-auto.sig @@ -1,3 +1 @@ -_̊П‡åNþÌ*–¿”ŽšJsµ¤‰Q…j ÜS¢llªY3Q]Âê«x+/V`$4xM…ñï晾èUăÚí«Sí³•Pl ¶Hbƒ a<£{p~`ÐŒó&a1E@²Ú;RNiL¬J¹@îpÌÏwk!Š ‡&Æ, /dev/null 2>&1 ; then - export EXISTS="command -v" -elif which which > /dev/null 2>&1 ; then - export EXISTS="which" -else - error "Cannot find command nor which... please install one!" - exit 1 -fi - -# Certbot itself needs root access for almost all modes of operation. -# certbot-auto needs root access to bootstrap OS dependencies and install -# Certbot at a protected path so it can be safely run as root. To accomplish -# this, this script will attempt to run itself as root if it doesn't have the -# necessary privileges by using `sudo` or falling back to `su` if it is not -# available. The mechanism used to obtain root access can be set explicitly by -# setting the environment variable LE_AUTO_SUDO to 'sudo', 'su', 'su_sudo', -# 'SuSudo', or '' as used below. - -# Because the parameters in `su -c` has to be a string, -# we need to properly escape it. -SuSudo() { - args="" - # This `while` loop iterates over all parameters given to this function. - # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrapped in a pair of `'`, then appended to `$args` string - # For example, `echo "It's only 1\$\!"` will be escaped to: - # 'echo' 'It'"'"'s only 1$!' - # │ │└┼┘│ - # │ │ │ └── `'s only 1$!'` the literal string - # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings following it - # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself - while [ $# -ne 0 ]; do - args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " - shift - done - su root -c "$args" -} - -# Sets the environment variable SUDO to be the name of the program or function -# to call to get root access. If this script already has root privleges, SUDO -# is set to an empty string. The value in SUDO should be run with the command -# to called with root privileges as arguments. -SetRootAuthMechanism() { - SUDO="" - if [ -n "${LE_AUTO_SUDO+x}" ]; then - case "$LE_AUTO_SUDO" in - SuSudo|su_sudo|su) - SUDO=SuSudo - ;; - sudo) - SUDO="sudo -E" - ;; - '') - # If we're not running with root, don't check that this script can only - # be modified by system users and groups. - NO_PERMISSIONS_CHECK=1 - ;; - *) - error "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." - exit 1 - esac - say "Using preset root authorization mechanism '$LE_AUTO_SUDO'." - else - if test "`id -u`" -ne "0" ; then - if $EXISTS sudo 1>/dev/null 2>&1; then - SUDO="sudo -E" - else - say \"sudo\" is not available, will use \"su\" for installation steps... - SUDO=SuSudo - fi - fi - fi -} - -if [ "$1" = "--cb-auto-has-root" ]; then - shift 1 -else - SetRootAuthMechanism - if [ -n "$SUDO" ]; then - say "Requesting to rerun $0 with root privileges..." - $SUDO "$0" --cb-auto-has-root "$@" - exit 0 - fi -fi - -# Runs this script again with the given arguments. --cb-auto-has-root is added -# to the command line arguments to ensure we don't try to acquire root a -# second time. After the script is rerun, we exit the current script. -RerunWithArgs() { - "$0" --cb-auto-has-root "$@" - exit 0 -} - -BootstrapMessage() { - # Arguments: Platform name - say "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" -} - -ExperimentalBootstrap() { - # Arguments: Platform name, bootstrap function name - if [ "$DEBUG" = 1 ]; then - if [ "$2" != "" ]; then - BootstrapMessage $1 - $2 - fi - else - error "FATAL: $1 support is very experimental at present..." - error "if you would like to work on improving it, please ensure you have backups" - error "and then run this script again with the --debug flag!" - error "Alternatively, you can install OS dependencies yourself and run this script" - error "again with --no-bootstrap." - exit 1 - fi -} - -DeprecationBootstrap() { - # Arguments: Platform name, bootstrap function name - if [ "$DEBUG" = 1 ]; then - if [ "$2" != "" ]; then - BootstrapMessage $1 - $2 - fi - else - error "WARNING: certbot-auto support for this $1 is DEPRECATED!" - error "Please visit certbot.eff.org to learn how to download a version of" - error "Certbot that is packaged for your system. While an existing version" - error "of certbot-auto may work currently, we have stopped supporting updating" - error "system packages for your system. Please switch to a packaged version" - error "as soon as possible." - exit 1 - fi -} - -MIN_PYTHON_2_VERSION="2.7" -MIN_PYVER2=$(echo "$MIN_PYTHON_2_VERSION" | sed 's/\.//') -MIN_PYTHON_3_VERSION="3.6" -MIN_PYVER3=$(echo "$MIN_PYTHON_3_VERSION" | sed 's/\.//') -# Sets LE_PYTHON to Python version string and PYVER to the first two -# digits of the python version. -# MIN_PYVER and MIN_PYTHON_VERSION are also set by this function, and their -# values depend on if we try to use Python 3 or Python 2. -DeterminePythonVersion() { - # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python - # - # If no Python is found, PYVER is set to 0. - if [ "$USE_PYTHON_3" = 1 ]; then - MIN_PYVER=$MIN_PYVER3 - MIN_PYTHON_VERSION=$MIN_PYTHON_3_VERSION - for LE_PYTHON in "$LE_PYTHON" python3; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - else - MIN_PYVER=$MIN_PYVER2 - MIN_PYTHON_VERSION=$MIN_PYTHON_2_VERSION - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - fi - if [ "$?" != "0" ]; then - if [ "$1" != "NOCRASH" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 - else - PYVER=0 - return 0 - fi - fi - - PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') - if [ "$PYVER" -lt "$MIN_PYVER" ]; then - if [ "$1" != "NOCRASH" ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." - exit 1 - fi - fi -} - -{{ bootstrappers/deb_common.sh }} -{{ bootstrappers/rpm_common_base.sh }} -{{ bootstrappers/rpm_common.sh }} -{{ bootstrappers/rpm_python3_legacy.sh }} -{{ bootstrappers/rpm_python3.sh }} -{{ bootstrappers/suse_common.sh }} -{{ bootstrappers/arch_common.sh }} -{{ bootstrappers/gentoo_common.sh }} -{{ bootstrappers/free_bsd.sh }} -{{ bootstrappers/mac.sh }} -{{ bootstrappers/smartos.sh }} -{{ bootstrappers/mageia_common.sh }} - -# Set Bootstrap to the function that installs OS dependencies on this system -# and BOOTSTRAP_VERSION to the unique identifier for the current version of -# that function. If Bootstrap is set to a function that doesn't install any -# packages BOOTSTRAP_VERSION is not set. -if [ -f /etc/debian_version ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/mageia-release ]; then - # Mageia has both /etc/mageia-release and /etc/redhat-release - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/redhat-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 - # Run DeterminePythonVersion to decide on the basis of available Python versions - # whether to use 2.x or 3.x on RedHat-like systems. - # Then, revert LE_PYTHON to its previous state. - prev_le_python="$LE_PYTHON" - unset LE_PYTHON - DeterminePythonVersion "NOCRASH" - - RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"` - - if [ "$PYVER" -eq 26 -a $(uname -m) != 'x86_64' ]; then - # 32 bits CentOS 6 and affiliates are not supported anymore by certbot-auto. - DEPRECATED_OS=1 - fi - - # Set RPM_DIST_VERSION to VERSION_ID from /etc/os-release after splitting on - # '.' characters (e.g. "8.0" becomes "8"). If the command exits with an - # error, RPM_DIST_VERSION is set to "unknown". - RPM_DIST_VERSION=$( (. /etc/os-release 2> /dev/null && echo "$VERSION_ID") | cut -d '.' -f1 || echo "unknown") - - # If RPM_DIST_VERSION is an empty string or it contains any nonnumeric - # characters, the value is unexpected so we set RPM_DIST_VERSION to 0. - if [ -z "$RPM_DIST_VERSION" ] || [ -n "$(echo "$RPM_DIST_VERSION" | tr -d '[0-9]')" ]; then - RPM_DIST_VERSION=0 - fi - - # Handle legacy RPM distributions - if [ "$PYVER" -eq 26 ]; then - # Check if an automated bootstrap can be achieved on this system. - if ! Python36SclIsAvailable; then - INTERACTIVE_BOOTSTRAP=1 - fi - - USE_PYTHON_3=1 - - # Try now to enable SCL rh-python36 for systems already bootstrapped - # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto - EnablePython36SCL - else - # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. - # RHEL 8 also uses python3 by default. - if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 ]; then - RPM_USE_PYTHON_3=1 - elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then - RPM_USE_PYTHON_3=1 - elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then - RPM_USE_PYTHON_3=1 - else - RPM_USE_PYTHON_3=0 - fi - - if [ "$RPM_USE_PYTHON_3" = 1 ]; then - USE_PYTHON_3=1 - fi - fi - - LE_PYTHON="$prev_le_python" -elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/arch-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/manjaro-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/gentoo-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif uname | grep -iq FreeBSD ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif uname | grep -iq Darwin ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -else - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -fi - -# We handle this case after determining the normal bootstrap version to allow -# variables like USE_PYTHON_3 to be properly set. As described above, if the -# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not -# be set so we unset it here. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } - unset BOOTSTRAP_VERSION -fi - -if [ "$DEPRECATED_OS" = 1 ]; then - Bootstrap() { - error "Skipping bootstrap because certbot-auto is deprecated on this system." - } - unset BOOTSTRAP_VERSION -fi - -# Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used -# to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set -# if it is unknown how OS dependencies were installed on this system. -SetPrevBootstrapVersion() { - if [ -f $BOOTSTRAP_VERSION_PATH ]; then - PREV_BOOTSTRAP_VERSION=$(cat "$BOOTSTRAP_VERSION_PATH") - # The list below only contains bootstrap version strings that existed before - # we started writing them to disk. - # - # DO NOT MODIFY THIS LIST UNLESS YOU KNOW WHAT YOU'RE DOING! - elif grep -Fqx "$BOOTSTRAP_VERSION" << "UNLIKELY_EOF" -BootstrapDebCommon 1 -BootstrapMageiaCommon 1 -BootstrapRpmCommon 1 -BootstrapSuseCommon 1 -BootstrapArchCommon 1 -BootstrapGentooCommon 1 -BootstrapFreeBsd 1 -BootstrapMac 1 -BootstrapSmartOS 1 -UNLIKELY_EOF - then - # If there's no bootstrap version saved to disk, but the currently selected - # bootstrap script is from before we started saving the version number, - # return the currently selected version to prevent us from rebootstrapping - # unnecessarily. - PREV_BOOTSTRAP_VERSION="$BOOTSTRAP_VERSION" - fi -} - -TempDir() { - mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS -} - -# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, -# returns a non-zero number. -OldVenvExists() { - [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] -} - -# Given python path, version 1 and version 2, check if version 1 is outdated compared to version 2. -# An unofficial version provided as version 1 (eg. 0.28.0.dev0) will be treated -# specifically by printing "UNOFFICIAL". Otherwise, print "OUTDATED" if version 1 -# is outdated, and "UP_TO_DATE" if not. -# This function relies only on installed python environment (2.x or 3.x) by certbot-auto. -CompareVersions() { - "$1" - "$2" "$3" << "UNLIKELY_EOF" -import sys -from distutils.version import StrictVersion - -try: - current = StrictVersion(sys.argv[1]) -except ValueError: - sys.stdout.write('UNOFFICIAL') - sys.exit() - -try: - remote = StrictVersion(sys.argv[2]) -except ValueError: - sys.stdout.write('UP_TO_DATE') - sys.exit() - -if current < remote: - sys.stdout.write('OUTDATED') -else: - sys.stdout.write('UP_TO_DATE') -UNLIKELY_EOF -} - -# Create a new virtual environment for Certbot. It will overwrite any existing one. -# Parameters: LE_PYTHON, VENV_PATH, PYVER, VERBOSE -CreateVenv() { - "$1" - "$2" "$3" "$4" << "UNLIKELY_EOF" -{{ create_venv.py }} -UNLIKELY_EOF -} - -# Check that the given PATH_TO_CHECK has secured permissions. -# Parameters: LE_PYTHON, PATH_TO_CHECK -CheckPathPermissions() { - "$1" - "$2" << "UNLIKELY_EOF" -{{ check_permissions.py }} -UNLIKELY_EOF -} - -if [ "$1" = "--le-auto-phase2" ]; then - # Phase 2: Create venv, install LE, and run. - - shift 1 # the --le-auto-phase2 arg - - if [ "$DEPRECATED_OS" = 1 ]; then - # Phase 2 damage control mode for deprecated OSes. - # In this situation, we bypass any bootstrap or certbot venv setup. - error "Your system is not supported by certbot-auto anymore." - - if [ ! -d "$VENV_PATH" ] && OldVenvExists; then - VENV_BIN="$OLD_VENV_PATH/bin" - fi - - if [ -f "$VENV_BIN/letsencrypt" -a "$INSTALL_ONLY" != 1 ]; then - error "certbot-auto and its Certbot installation will no longer receive updates." - error "You will not receive any bug fixes including those fixing server compatibility" - error "or security problems." - error "Please visit https://certbot.eff.org/ to check for other alternatives." - "$VENV_BIN/letsencrypt" "$@" - exit 0 - else - error "Certbot cannot be installed." - error "Please visit https://certbot.eff.org/ to check for other alternatives." - exit 1 - fi - fi - - SetPrevBootstrapVersion - - if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then - unset LE_PYTHON - fi - - INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ] || OldVenvExists; then - # If the selected Bootstrap function isn't a noop and it differs from the - # previously used version - if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then - # Check if we can rebootstrap without manual user intervention: this requires that - # certbot-auto is in non-interactive mode AND selected bootstrap does not claim to - # require a manual user intervention. - if [ "$NONINTERACTIVE" = 1 -a "$INTERACTIVE_BOOTSTRAP" != 1 ]; then - CAN_REBOOTSTRAP=1 - fi - # Check if rebootstrap can be done non-interactively and current shell is non-interactive - # (true if stdin and stdout are not attached to a terminal). - if [ \( "$CAN_REBOOTSTRAP" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - if [ -d "$VENV_PATH" ]; then - rm -rf "$VENV_PATH" - fi - # In the case the old venv was just a symlink to the new one, - # OldVenvExists is now false because we deleted the venv at VENV_PATH. - if OldVenvExists; then - rm -rf "$OLD_VENV_PATH" - ln -s "$VENV_PATH" "$OLD_VENV_PATH" - fi - RerunWithArgs "$@" - # Otherwise bootstrap needs to be done manually by the user. - else - # If it is because bootstrapping is interactive, --non-interactive will be of no use. - if [ "$INTERACTIVE_BOOTSTRAP" = 1 ]; then - error "Skipping upgrade because new OS dependencies may need to be installed." - error "This requires manual user intervention: please run this script again manually." - # If this is because of the environment (eg. non interactive shell without - # --non-interactive flag set), help the user in that direction. - else - error "Skipping upgrade because new OS dependencies may need to be installed." - error - error "To upgrade to a newer version, please run this script again manually so you can" - error "approve changes or with --non-interactive on the command line to automatically" - error "install any required packages." - fi - # Set INSTALLED_VERSION to be the same so we don't update the venv - INSTALLED_VERSION="$LE_AUTO_VERSION" - # Continue to use OLD_VENV_PATH if the new venv doesn't exist - if [ ! -d "$VENV_PATH" ]; then - VENV_BIN="$OLD_VENV_PATH/bin" - fi - fi - elif [ -f "$VENV_BIN/letsencrypt" ]; then - # --version output ran through grep due to python-cryptography DeprecationWarnings - # grep for both certbot and letsencrypt until certbot and shim packages have been released - INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) - if [ -z "$INSTALLED_VERSION" ]; then - error "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 - "$VENV_BIN/letsencrypt" --version - exit 1 - fi - fi - fi - - if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then - say "Creating virtual environment..." - DeterminePythonVersion - CreateVenv "$LE_PYTHON" "$VENV_PATH" "$PYVER" "$VERBOSE" - - if [ -n "$BOOTSTRAP_VERSION" ]; then - echo "$BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH" - elif [ -n "$PREV_BOOTSTRAP_VERSION" ]; then - echo "$PREV_BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH" - fi - - say "Installing Python packages..." - TEMP_DIR=$(TempDir) - trap 'rm -rf "$TEMP_DIR"' EXIT - # There is no $ interpolation due to quotes on starting heredoc delimiter. - # ------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" -{{ dependency-requirements.txt }} -{{ letsencrypt-requirements.txt }} -{{ certbot-requirements.txt }} -UNLIKELY_EOF - # ------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" -{{ pipstrap.py }} -UNLIKELY_EOF - # ------------------------------------------------------------------------- - # Set PATH so pipstrap upgrades the right (v)env: - PATH="$VENV_BIN:$PATH" "$VENV_BIN/python" "$TEMP_DIR/pipstrap.py" - set +e - if [ "$VERBOSE" = 1 ]; then - "$VENV_BIN/pip" install --disable-pip-version-check --no-cache-dir --require-hashes -r "$TEMP_DIR/letsencrypt-auto-requirements.txt" - else - PIP_OUT=`"$VENV_BIN/pip" install --disable-pip-version-check --no-cache-dir --require-hashes -r "$TEMP_DIR/letsencrypt-auto-requirements.txt" 2>&1` - fi - PIP_STATUS=$? - set -e - if [ "$PIP_STATUS" != 0 ]; then - # Report error. (Otherwise, be quiet.) - error "Had a problem while installing Python packages." - if [ "$VERBOSE" != 1 ]; then - error - error "pip prints the following errors: " - error "=====================================================" - error "$PIP_OUT" - error "=====================================================" - error - error "Certbot has problem setting up the virtual environment." - - if `echo $PIP_OUT | grep -q Killed` || `echo $PIP_OUT | grep -q "allocate memory"` ; then - error - error "Based on your pip output, the problem can likely be fixed by " - error "increasing the available memory." - else - error - error "We were not be able to guess the right solution from your pip " - error "output." - fi - - error - error "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" - error "for possible solutions." - error "You may also find some support resources at https://certbot.eff.org/support/ ." - fi - rm -rf "$VENV_PATH" - exit 1 - fi - - if [ -d "$OLD_VENV_PATH" -a ! -L "$OLD_VENV_PATH" ]; then - rm -rf "$OLD_VENV_PATH" - ln -s "$VENV_PATH" "$OLD_VENV_PATH" - fi - - say "Installation succeeded." - fi - - # If you're modifying any of the code after this point in this current `if` block, you - # may need to update the "$DEPRECATED_OS" = 1 case at the beginning of phase 2 as well. - - if [ "$INSTALL_ONLY" = 1 ]; then - say "Certbot is installed." - exit 0 - fi - - "$VENV_BIN/letsencrypt" "$@" - -else - # Phase 1: Upgrade certbot-auto if necessary, then self-invoke. - # - # Each phase checks the version of only the thing it is responsible for - # upgrading. Phase 1 checks the version of the latest release of - # certbot-auto (which is always the same as that of the certbot - # package). Phase 2 checks the version of the locally installed certbot. - export PHASE_1_VERSION="$LE_AUTO_VERSION" - - if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if ! OldVenvExists; then - if [ "$HELP" = 1 ]; then - echo "$USAGE" - exit 0 - fi - # If it looks like we've never bootstrapped before, bootstrap: - Bootstrap - fi - fi - if [ "$OS_PACKAGES_ONLY" = 1 ]; then - say "OS packages installed." - exit 0 - fi - - DeterminePythonVersion "NOCRASH" - # Don't warn about file permissions if the user disabled the check or we - # can't find an up-to-date Python. - if [ "$PYVER" -ge "$MIN_PYVER" -a "$NO_PERMISSIONS_CHECK" != 1 ]; then - # If the script fails for some reason, don't break certbot-auto. - set +e - # Suppress unexpected error output. - CHECK_PERM_OUT=$(CheckPathPermissions "$LE_PYTHON" "$0" 2>/dev/null) - CHECK_PERM_STATUS="$?" - set -e - # Only print output if the script ran successfully and it actually produced - # output. The latter check resolves - # https://github.com/certbot/certbot/issues/7012. - if [ "$CHECK_PERM_STATUS" = 0 -a -n "$CHECK_PERM_OUT" ]; then - error "$CHECK_PERM_OUT" - fi - fi - - if [ "$NO_SELF_UPGRADE" != 1 ]; then - TEMP_DIR=$(TempDir) - trap 'rm -rf "$TEMP_DIR"' EXIT - # --------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" -{{ fetch.py }} -UNLIKELY_EOF - # --------------------------------------------------------------------------- - if [ "$PYVER" -lt "$MIN_PYVER" ]; then - error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." - elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then - error "WARNING: unable to check for updates." - fi - - # If for any reason REMOTE_VERSION is not set, let's assume certbot-auto is up-to-date, - # and do not go into the self-upgrading process. - if [ -n "$REMOTE_VERSION" ]; then - LE_VERSION_STATE=`CompareVersions "$LE_PYTHON" "$LE_AUTO_VERSION" "$REMOTE_VERSION"` - - if [ "$LE_VERSION_STATE" = "UNOFFICIAL" ]; then - say "Unofficial certbot-auto version detected, self-upgrade is disabled: $LE_AUTO_VERSION" - elif [ "$LE_VERSION_STATE" = "OUTDATED" ]; then - say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." - - # Now we drop into Python so we don't have to install even more - # dependencies (curl, etc.), for better flow control, and for the option of - # future Windows compatibility. - "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" - - # Install new copy of certbot-auto. - # TODO: Deal with quotes in pathnames. - say "Replacing certbot-auto..." - # Clone permissions with cp. chmod and chown don't have a --reference - # option on macOS or BSD, and stat -c on Linux is stat -f on macOS and BSD: - cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" - cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" - # Using mv rather than cp leaves the old file descriptor pointing to the - # original copy so the shell can continue to read it unmolested. mv across - # filesystems is non-atomic, doing `rm dest, cp src dest, rm src`, but the - # cp is unlikely to fail if the rm doesn't. - mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" - fi # A newer version is available. - fi - fi # Self-upgrading is allowed. - - RerunWithArgs --le-auto-phase2 "$@" -fi diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh deleted file mode 100755 index 3be78d3f8..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ /dev/null @@ -1,37 +0,0 @@ -# If new packages are installed by BootstrapArchCommon below, this version -# number must be increased. -BOOTSTRAP_ARCH_COMMON_VERSION=1 - -BootstrapArchCommon() { - # Tested with: - # - ArchLinux (x86_64) - # - # "python-virtualenv" is Python3, but "python2-virtualenv" provides - # only "virtualenv2" binary, not "virtualenv". - - deps=" - python2 - python-virtualenv - gcc - augeas - openssl - libffi - ca-certificates - pkg-config - " - - # pacman -T exits with 127 if there are missing dependencies - missing=$(pacman -T $deps) || true - - if [ "$ASSUME_YES" = 1 ]; then - noconfirm="--noconfirm" - fi - - if [ "$missing" ]; then - if [ "$QUIET" = 1 ]; then - pacman -S --needed $missing $noconfirm > /dev/null - else - pacman -S --needed $missing $noconfirm - fi - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh deleted file mode 100644 index 93bdc63b4..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ /dev/null @@ -1,67 +0,0 @@ -# If new packages are installed by BootstrapDebCommon below, this version -# number must be increased. -BOOTSTRAP_DEB_COMMON_VERSION=1 - -BootstrapDebCommon() { - # Current version tested with: - # - # - Ubuntu - # - 14.04 (x64) - # - 15.04 (x64) - # - Debian - # - 7.9 "wheezy" (x64) - # - sid (2015-10-21) (x64) - - # Past versions tested with: - # - # - Debian 8.0 "jessie" (x64) - # - Raspbian 7.8 (armhf) - - # Believed not to work: - # - # - Debian 6.0.10 "squeeze" (x64) - - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='-qq' - fi - - apt-get $QUIET_FLAG update || error apt-get update hit problems but continuing anyway... - - # virtualenv binary can be found in different packages depending on - # distro version (#346) - - virtualenv= - # virtual env is known to apt and is installable - if apt-cache show virtualenv > /dev/null 2>&1 ; then - if ! LC_ALL=C apt-cache --quiet=0 show virtualenv 2>&1 | grep -q 'No packages found'; then - virtualenv="virtualenv" - fi - fi - - if apt-cache show python-virtualenv > /dev/null 2>&1; then - virtualenv="$virtualenv python-virtualenv" - fi - - augeas_pkg="libaugeas0 augeas-lenses" - - if [ "$ASSUME_YES" = 1 ]; then - YES_FLAG="-y" - fi - - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ - python \ - python-dev \ - $virtualenv \ - gcc \ - $augeas_pkg \ - libssl-dev \ - openssl \ - libffi-dev \ - ca-certificates \ - - - if ! $EXISTS virtualenv > /dev/null ; then - error Failed to install a working \"virtualenv\" command, exiting - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh b/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh deleted file mode 100755 index a67c85619..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh +++ /dev/null @@ -1,15 +0,0 @@ -# If new packages are installed by BootstrapFreeBsd below, this version number -# must be increased. -BOOTSTRAP_FREEBSD_VERSION=1 - -BootstrapFreeBsd() { - if [ "$QUIET" = 1 ]; then - QUIET_FLAG="--quiet" - fi - - pkg install -Ay $QUIET_FLAG \ - python \ - py27-virtualenv \ - augeas \ - libffi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh deleted file mode 100755 index e2d24b5fb..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh +++ /dev/null @@ -1,31 +0,0 @@ -# If new packages are installed by BootstrapGentooCommon below, this version -# number must be increased. -BOOTSTRAP_GENTOO_COMMON_VERSION=1 - -BootstrapGentooCommon() { - PACKAGES=" - dev-lang/python:2.7 - dev-python/virtualenv - app-admin/augeas - dev-libs/openssl - dev-libs/libffi - app-misc/ca-certificates - virtual/pkgconfig" - - ASK_OPTION="--ask" - if [ "$ASSUME_YES" = 1 ]; then - ASK_OPTION="" - fi - - case "$PACKAGE_MANAGER" in - (paludis) - cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x - ;; - (pkgcore) - pmerge --noreplace --oneshot $ASK_OPTION $PACKAGES - ;; - (portage|*) - emerge --noreplace --oneshot $ASK_OPTION $PACKAGES - ;; - esac -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh deleted file mode 100755 index 9e26d3389..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh +++ /dev/null @@ -1,48 +0,0 @@ -# If new packages are installed by BootstrapMac below, this version number must -# be increased. -BOOTSTRAP_MAC_VERSION=1 - -BootstrapMac() { - if hash brew 2>/dev/null; then - say "Using Homebrew to install dependencies..." - pkgman=brew - pkgcmd="brew install" - elif hash port 2>/dev/null; then - say "Using MacPorts to install dependencies..." - pkgman=port - pkgcmd="port install" - else - say "No Homebrew/MacPorts; installing Homebrew..." - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - pkgman=brew - pkgcmd="brew install" - fi - - $pkgcmd augeas - if [ "$(which python)" = "/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python" \ - -o "$(which python)" = "/usr/bin/python" ]; then - # We want to avoid using the system Python because it requires root to use pip. - # python.org, MacPorts or HomeBrew Python installations should all be OK. - say "Installing python..." - $pkgcmd python - fi - - # Workaround for _dlopen not finding augeas on macOS - if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then - say "Applying augeas workaround" - mkdir -p /usr/local/lib/ - ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ - fi - - if ! hash pip 2>/dev/null; then - say "pip not installed" - say "Installing pip..." - curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python - fi - - if ! hash virtualenv 2>/dev/null; then - say "virtualenv not installed." - say "Installing with pip..." - pip install virtualenv - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh deleted file mode 100644 index dfa5b47f3..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh +++ /dev/null @@ -1,30 +0,0 @@ -# If new packages are installed by BootstrapMageiaCommon below, this version -# number must be increased. -BOOTSTRAP_MAGEIA_COMMON_VERSION=1 - -BootstrapMageiaCommon() { - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='--quiet' - fi - - if ! urpmi --force $QUIET_FLAG \ - python \ - libpython-devel \ - python-virtualenv - then - error "Could not install Python dependencies. Aborting bootstrap!" - exit 1 - fi - - if ! urpmi --force $QUIET_FLAG \ - git \ - gcc \ - python-augeas \ - libopenssl-devel \ - libffi-devel \ - rootcerts - then - error "Could not install additional dependencies. Aborting bootstrap!" - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh deleted file mode 100755 index 80d55a393..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ /dev/null @@ -1,45 +0,0 @@ -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 - - InitializeRPMCommonBase - - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $TOOL list python >/dev/null 2>&1; then - python_pkgs="$python - python-devel - python-virtualenv - python-tools - python-pip - " - # Fedora 26 starts to use the prefix python2 for python2 based packages. - # this elseif is theoretically for any Fedora over version 26: - elif $TOOL list python2 >/dev/null 2>&1; then - python_pkgs="$python2 - python2-libs - python2-setuptools - python2-devel - python2-virtualenv - python2-tools - python2-pip - " - # Some distros and older versions of current distros use a "python27" - # instead of the "python" or "python-" naming convention. - else - python_pkgs="$python27 - python27-devel - python27-virtualenv - python27-tools - python27-pip - " - fi - - BootstrapRpmCommonBase "$python_pkgs" -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh deleted file mode 100644 index 2b00b199b..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh +++ /dev/null @@ -1,60 +0,0 @@ -# If new packages are installed by BootstrapRpmCommonBase below, version -# numbers in rpm_common.sh and rpm_python3.sh must be increased. - -# Sets TOOL to the name of the package manager -# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. -# Note: this function is called both while selecting the bootstrap scripts and -# during the actual bootstrap. Some things like prompting to user can be done in the latter -# case, but not in the former one. -InitializeRPMCommonBase() { - if type dnf 2>/dev/null - then - TOOL=dnf - elif type yum 2>/dev/null - then - TOOL=yum - - else - error "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 - fi - - if [ "$ASSUME_YES" = 1 ]; then - YES_FLAG="-y" - fi - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='--quiet' - fi -} - -BootstrapRpmCommonBase() { - # Arguments: whitespace-delimited python packages to install - - InitializeRPMCommonBase # This call is superfluous in practice - - pkgs=" - gcc - augeas-libs - openssl - openssl-devel - libffi-devel - redhat-rpm-config - ca-certificates - " - - # Add the python packages - pkgs="$pkgs - $1 - " - - if $TOOL list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi - - if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh deleted file mode 100644 index ac0553db5..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh +++ /dev/null @@ -1,23 +0,0 @@ -# If new packages are installed by BootstrapRpmPython3 below, this version -# number must be increased. -BOOTSTRAP_RPM_PYTHON3_VERSION=1 - -BootstrapRpmPython3() { - # Tested with: - # - Fedora 29 - - InitializeRPMCommonBase - - # Fedora 29 must use python3-virtualenv - if $TOOL list python3-virtualenv >/dev/null 2>&1; then - python_pkgs="python3 - python3-virtualenv - python3-devel - " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 - fi - - BootstrapRpmCommonBase "$python_pkgs" -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh deleted file mode 100644 index febfc7a83..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh +++ /dev/null @@ -1,78 +0,0 @@ -# If new packages are installed by BootstrapRpmPython3 below, this version -# number must be increased. -BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION=1 - -# Checks if rh-python36 can be installed. -Python36SclIsAvailable() { - InitializeRPMCommonBase >/dev/null 2>&1; - - if "${TOOL}" list rh-python36 >/dev/null 2>&1; then - return 0 - fi - if "${TOOL}" list centos-release-scl >/dev/null 2>&1; then - return 0 - fi - return 1 -} - -# Try to enable rh-python36 from SCL if it is necessary and possible. -EnablePython36SCL() { - if "$EXISTS" python3.6 > /dev/null 2> /dev/null; then - return 0 - fi - if [ ! -f /opt/rh/rh-python36/enable ]; then - return 0 - fi - set +e - if ! . /opt/rh/rh-python36/enable; then - error 'Unable to enable rh-python36!' - exit 1 - fi - set -e -} - -# This bootstrap concerns old RedHat-based distributions that do not ship by default -# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing -# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6. -BootstrapRpmPython3Legacy() { - # Tested with: - # - CentOS 6 - - InitializeRPMCommonBase - - if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then - echo "To use Certbot on this operating system, packages from the SCL repository need to be installed." - if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then - error "Enable the SCL repository and try running Certbot again." - exit 1 - fi - if [ "${ASSUME_YES}" = 1 ]; then - /bin/echo -n "Enabling the SCL repository in 3 seconds... (Press Ctrl-C to cancel)" - sleep 1s - /bin/echo -ne "\e[0K\rEnabling the SCL repository in 2 seconds... (Press Ctrl-C to cancel)" - sleep 1s - /bin/echo -e "\e[0K\rEnabling the SCL repository in 1 second... (Press Ctrl-C to cancel)" - sleep 1s - fi - if ! "${TOOL}" install "${YES_FLAG}" "${QUIET_FLAG}" centos-release-scl; then - error "Could not enable SCL. Aborting bootstrap!" - exit 1 - fi - fi - - # CentOS 6 must use rh-python36 from SCL - if "${TOOL}" list rh-python36 >/dev/null 2>&1; then - python_pkgs="rh-python36-python - rh-python36-python-virtualenv - rh-python36-python-devel - " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 - fi - - BootstrapRpmCommonBase "${python_pkgs}" - - # Enable SCL rh-python36 after bootstrapping. - EnablePython36SCL -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh b/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh deleted file mode 100644 index ac7c0ed4a..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh +++ /dev/null @@ -1,8 +0,0 @@ -# If new packages are installed by BootstrapSmartOS below, this version number -# must be increased. -BOOTSTRAP_SMARTOS_VERSION=1 - -BootstrapSmartOS() { - pkgin update - pkgin -y install 'gcc49' 'py27-augeas' 'py27-virtualenv' -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh deleted file mode 100755 index 7fa28ce50..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh +++ /dev/null @@ -1,36 +0,0 @@ -# If new packages are installed by BootstrapSuseCommon below, this version -# number must be increased. -BOOTSTRAP_SUSE_COMMON_VERSION=1 - -BootstrapSuseCommon() { - # SLE12 don't have python-virtualenv - - if [ "$ASSUME_YES" = 1 ]; then - zypper_flags="-nq" - install_flags="-l" - fi - - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='-qq' - fi - - if zypper search -x python-virtualenv >/dev/null 2>&1; then - OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" - else - # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv - # is a source package, and python2-virtualenv must be used instead. - # Also currently python2-setuptools is not a dependency of python2-virtualenv, - # while it should be. Installing it explicitly until upstream fix. - OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" - fi - - zypper $QUIET_FLAG $zypper_flags in $install_flags \ - python \ - python-devel \ - $OPENSUSE_VIRTUALENV_PACKAGES \ - gcc \ - augeas-lenses \ - libopenssl-devel \ - libffi-devel \ - ca-certificates -} diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt deleted file mode 100644 index 0f51c4b8a..000000000 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -certbot==1.13.0 \ - --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ - --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 -acme==1.13.0 \ - --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ - --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 -certbot-apache==1.13.0 \ - --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ - --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 -certbot-nginx==1.13.0 \ - --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ - --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 diff --git a/letsencrypt-auto-source/pieces/check_permissions.py b/letsencrypt-auto-source/pieces/check_permissions.py deleted file mode 100644 index ba55e6d97..000000000 --- a/letsencrypt-auto-source/pieces/check_permissions.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Verifies certbot-auto cannot be modified by unprivileged users. - -This script takes the path to certbot-auto as its only command line -argument. It then checks that the file can only be modified by uid/gid -< 1000 and if other users can modify the file, it prints a warning with -a suggestion on how to solve the problem. - -Permissions on symlinks in the absolute path of certbot-auto are ignored -and only the canonical path to certbot-auto is checked. There could be -permissions problems due to the symlinks that are unreported by this -script, however, issues like this were not caused by our documentation -and are ignored for the sake of simplicity. - -All warnings are printed to stdout rather than stderr so all stderr -output from this script can be suppressed to avoid printing messages if -this script fails for some reason. - -""" -from __future__ import print_function - -import os -import stat -import sys - - -FORUM_POST_URL = 'https://community.letsencrypt.org/t/certbot-auto-deployment-best-practices/91979/' - - -def has_safe_permissions(path): - """Returns True if the given path has secure permissions. - - The permissions are considered safe if the file is only writable by - uid/gid < 1000. - - The reason we allow more IDs than 0 is because on some systems such - as Debian, system users/groups other than uid/gid 0 are used for the - path we recommend in our instructions which is /usr/local/bin. 1000 - was chosen because on Debian 0-999 is reserved for system IDs[1] and - on RHEL either 0-499 or 0-999 is reserved depending on the - version[2][3]. Due to these differences across different OSes, this - detection isn't perfect so we only determine permissions are - insecure when we can be reasonably confident there is a problem - regardless of the underlying OS. - - [1] https://www.debian.org/doc/debian-policy/ch-opersys.html#uid-and-gid-classes - [2] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/ch-managing_users_and_groups - [3] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/system_administrators_guide/ch-managing_users_and_groups - - :param str path: filesystem path to check - :returns: True if the path has secure permissions, otherwise, False - :rtype: bool - - """ - # os.stat follows symlinks before obtaining information about a file. - stat_result = os.stat(path) - if stat_result.st_mode & stat.S_IWOTH: - return False - if stat_result.st_mode & stat.S_IWGRP and stat_result.st_gid >= 1000: - return False - if stat_result.st_mode & stat.S_IWUSR and stat_result.st_uid >= 1000: - return False - return True - - -def main(certbot_auto_path): - current_path = os.path.realpath(certbot_auto_path) - last_path = None - permissions_ok = True - # This loop makes use of the fact that os.path.dirname('/') == '/'. - while current_path != last_path and permissions_ok: - permissions_ok = has_safe_permissions(current_path) - last_path = current_path - current_path = os.path.dirname(current_path) - - if not permissions_ok: - print('{0} has insecure permissions!'.format(certbot_auto_path)) - print('To learn how to fix them, visit {0}'.format(FORUM_POST_URL)) - - -if __name__ == '__main__': - main(sys.argv[1]) diff --git a/letsencrypt-auto-source/pieces/create_venv.py b/letsencrypt-auto-source/pieces/create_venv.py deleted file mode 100755 index a618e228a..000000000 --- a/letsencrypt-auto-source/pieces/create_venv.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -import os -import shutil -import subprocess -import sys - - -def create_venv(venv_path, pyver, verbose): - if os.path.exists(venv_path): - shutil.rmtree(venv_path) - - stdout = sys.stdout if verbose == '1' else open(os.devnull, 'w') - - if int(pyver) <= 27: - # Use virtualenv binary - environ = os.environ.copy() - environ['VIRTUALENV_NO_DOWNLOAD'] = '1' - command = ['virtualenv', '--no-site-packages', '--python', sys.executable, venv_path] - subprocess.check_call(command, stdout=stdout, env=environ) - else: - # Use embedded venv module in Python 3 - command = [sys.executable, '-m', 'venv', venv_path] - subprocess.check_call(command, stdout=stdout) - - -if __name__ == '__main__': - create_venv(*sys.argv[1:]) diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt deleted file mode 100644 index f7a517e06..000000000 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ /dev/null @@ -1,264 +0,0 @@ -# This is the flattened list of packages certbot-auto installs. -# To generate this, do (with docker and package hashin installed): -# ``` -# letsencrypt-auto-source/rebuild_dependencies.py \ -# letsencrypt-auto-source/pieces/dependency-requirements.txt -# ``` -# If you want to update a single dependency, run commands similar to these: -# ``` -# pip install hashin -# hashin -r dependency-requirements.txt cryptography==1.5.2 -# ``` -ConfigArgParse==1.2.3 \ - --hash=sha256:edd17be986d5c1ba2e307150b8e5f5107aba125f3574dddd02c85d5cdcfd37dc -certifi==2020.4.5.1 \ - --hash=sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304 \ - --hash=sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519 -cffi==1.14.0 \ - --hash=sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff \ - --hash=sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b \ - --hash=sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac \ - --hash=sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0 \ - --hash=sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384 \ - --hash=sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26 \ - --hash=sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6 \ - --hash=sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b \ - --hash=sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e \ - --hash=sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd \ - --hash=sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2 \ - --hash=sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66 \ - --hash=sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc \ - --hash=sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8 \ - --hash=sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55 \ - --hash=sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4 \ - --hash=sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5 \ - --hash=sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d \ - --hash=sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78 \ - --hash=sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa \ - --hash=sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793 \ - --hash=sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f \ - --hash=sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a \ - --hash=sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f \ - --hash=sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30 \ - --hash=sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f \ - --hash=sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3 \ - --hash=sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c -chardet==3.0.4 \ - --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ - --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 -configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 -cryptography==2.8 \ - --hash=sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c \ - --hash=sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595 \ - --hash=sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad \ - --hash=sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651 \ - --hash=sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2 \ - --hash=sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff \ - --hash=sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d \ - --hash=sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42 \ - --hash=sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d \ - --hash=sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e \ - --hash=sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912 \ - --hash=sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793 \ - --hash=sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13 \ - --hash=sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7 \ - --hash=sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0 \ - --hash=sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879 \ - --hash=sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f \ - --hash=sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9 \ - --hash=sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2 \ - --hash=sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf \ - --hash=sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8 -distro==1.5.0 \ - --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ - --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 -enum34==1.1.10; python_version < '3.4' \ - --hash=sha256:a98a201d6de3f2ab3db284e70a33b0f896fbf35f8086594e8c9e74b909058d53 \ - --hash=sha256:c3858660960c984d6ab0ebad691265180da2b43f07e061c0f8dca9ef3cffd328 \ - --hash=sha256:cce6a7477ed816bd2542d03d53db9f0db935dd013b70f336a95c73979289f248 -funcsigs==1.0.2 \ - --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ - --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 -idna==2.9 \ - --hash=sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb \ - --hash=sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa -ipaddress==1.0.23 \ - --hash=sha256:6e0f4a39e66cb5bb9a137b00276a2eff74f93b71dcbdad6f10ff7df9d3557fcc \ - --hash=sha256:b7f8e0369580bb4a24d5ba1d7cc29660a4a6987763faf1d8a8046830e020e7e2 -josepy==1.3.0 \ - --hash=sha256:c341ffa403399b18e9eae9012f804843045764d1390f9cb4648980a7569b1619 \ - --hash=sha256:e54882c64be12a2a76533f73d33cba9e331950fda9e2731e843490b774e7a01c -mock==1.3.0 \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb -parsedatetime==2.5 \ - --hash=sha256:3b835fc54e472c17ef447be37458b400e3fefdf14bb1ffdedb5d2c853acf4ba1 \ - --hash=sha256:d2e9ddb1e463de871d32088a3f3cea3dc8282b1b2800e081bd0ef86900451667 -pbr==5.4.5 \ - --hash=sha256:07f558fece33b05caf857474a366dfcc00562bca13dd8b47b2b3e22d9f9bf55c \ - --hash=sha256:579170e23f8e0c2f24b0de612f71f648eccb79fb1322c814ae6b3c07b5ba23e8 -pyOpenSSL==19.1.0 \ - --hash=sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504 \ - --hash=sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507 -pyRFC3339==1.1 \ - --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ - --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a -pycparser==2.20 \ - --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ - --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 -pyparsing==2.4.7 \ - --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ - --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b -python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 -pytz==2020.1 \ - --hash=sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed \ - --hash=sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048 -requests==2.23.0 \ - --hash=sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee \ - --hash=sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6 -requests-toolbelt==0.9.1 \ - --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ - --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 -six==1.15.0 \ - --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ - --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced -urllib3==1.25.9 \ - --hash=sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527 \ - --hash=sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115 -zope.component==4.6.1 \ - --hash=sha256:bfbe55d4a93e70a78b10edc3aad4de31bb8860919b7cbd8d66f717f7d7b279ac \ - --hash=sha256:d9c7c27673d787faff8a83797ce34d6ebcae26a370e25bddb465ac2182766aca -zope.deferredimport==4.3.1 \ - --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \ - --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a -zope.deprecation==4.4.0 \ - --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ - --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 -zope.event==4.4 \ - --hash=sha256:69c27debad9bdacd9ce9b735dad382142281ac770c4a432b533d6d65c4614bcf \ - --hash=sha256:d8e97d165fd5a0997b45f5303ae11ea3338becfe68c401dd88ffd2113fe5cae7 -zope.hookable==5.0.1 \ - --hash=sha256:0194b9b9e7f614abba60c90b231908861036578297515d3d6508eb10190f266d \ - --hash=sha256:0c2977473918bdefc6fa8dfb311f154e7f13c6133957fe649704deca79b92093 \ - --hash=sha256:17b8bdb3b77e03a152ca0d5ca185a7ae0156f5e5a2dbddf538676633a1f7380f \ - --hash=sha256:29d07681a78042cdd15b268ae9decffed9ace68a53eebeb61d65ae931d158841 \ - --hash=sha256:36fb1b35d1150267cb0543a1ddd950c0bc2c75ed0e6e92e3aaa6ac2e29416cb7 \ - --hash=sha256:3aed60c2bb5e812bbf9295c70f25b17ac37c233f30447a96c67913ba5073642f \ - --hash=sha256:3cac1565cc768911e72ca9ec4ddf5c5109e1fef0104f19f06649cf1874943b60 \ - --hash=sha256:3d4bc0cc4a37c3cd3081063142eeb2125511db3c13f6dc932d899c512690378e \ - --hash=sha256:3f73096f27b8c28be53ffb6604f7b570fbbb82f273c6febe5f58119009b59898 \ - --hash=sha256:522d1153d93f2d48aa0bd9fb778d8d4500be2e4dcf86c3150768f0e3adbbc4ef \ - --hash=sha256:523d2928fb7377bbdbc9af9c0b14ad73e6eaf226349f105733bdae27efd15b5a \ - --hash=sha256:5848309d4fc5c02150a45e8f8d2227e5bfda386a508bbd3160fed7c633c5a2fa \ - --hash=sha256:6781f86e6d54a110980a76e761eb54590630fd2af2a17d7edf02a079d2646c1d \ - --hash=sha256:6fd27921ebf3aaa945fa25d790f1f2046204f24dba4946f82f5f0a442577c3e9 \ - --hash=sha256:70d581862863f6bf9e175e85c9d70c2d7155f53fb04dcdb2f73cf288ca559a53 \ - --hash=sha256:81867c23b0dc66c8366f351d00923f2bc5902820a24c2534dfd7bf01a5879963 \ - --hash=sha256:81db29edadcbb740cd2716c95a297893a546ed89db1bfe9110168732d7f0afdd \ - --hash=sha256:86bd12624068cea60860a0759af5e2c3adc89c12aef6f71cf12f577e28deefe3 \ - --hash=sha256:9c184d8f9f7a76e1ced99855ccf390ffdd0ec3765e5cbf7b9cada600accc0a1e \ - --hash=sha256:acc789e8c29c13555e43fe4bf9fcd15a65512c9645e97bbaa5602e3201252b02 \ - --hash=sha256:afaa740206b7660d4cc3b8f120426c85761f51379af7a5b05451f624ad12b0af \ - --hash=sha256:b5f5fa323f878bb16eae68ea1ba7f6c0419d4695d0248bed4b18f51d7ce5ab85 \ - --hash=sha256:bd89e0e2c67bf4ac3aca2a19702b1a37269fb1923827f68324ac2e7afd6e3406 \ - --hash=sha256:c212de743283ec0735db24ec6ad913758df3af1b7217550ff270038062afd6ae \ - --hash=sha256:ca553f524293a0bdea05e7f44c3e685e4b7b022cb37d87bc4a3efa0f86587a8d \ - --hash=sha256:cab67065a3db92f636128d3157cc5424a145f82d96fb47159c539132833a6d36 \ - --hash=sha256:d3b3b3eedfdbf6b02898216e85aa6baf50207f4378a2a6803d6d47650cd37031 \ - --hash=sha256:d9f4a5a72f40256b686d31c5c0b1fde503172307beb12c1568296e76118e402c \ - --hash=sha256:df5067d87aaa111ed5d050e1ee853ba284969497f91806efd42425f5348f1c06 \ - --hash=sha256:e2587644812c6138f05b8a41594a8337c6790e3baf9a01915e52438c13fc6bef \ - --hash=sha256:e27fd877662db94f897f3fd532ef211ca4901eb1a70ba456f15c0866a985464a \ - --hash=sha256:e427ebbdd223c72e06ba94c004bb04e996c84dec8a0fa84e837556ae145c439e \ - --hash=sha256:e583ad4309c203ef75a09d43434cf9c2b4fa247997ecb0dcad769982c39411c7 \ - --hash=sha256:e760b2bc8ece9200804f0c2b64d10147ecaf18455a2a90827fbec4c9d84f3ad5 \ - --hash=sha256:ea9a9cc8bcc70e18023f30fa2f53d11ae069572a162791224e60cd65df55fb69 \ - --hash=sha256:ecb3f17dce4803c1099bd21742cd126b59817a4e76a6544d31d2cca6e30dbffd \ - --hash=sha256:ed794e3b3de42486d30444fb60b5561e724ee8a2d1b17b0c2e0f81e3ddaf7a87 \ - --hash=sha256:ee885d347279e38226d0a437b6a932f207f691c502ee565aba27a7022f1285df \ - --hash=sha256:fd5e7bc5f24f7e3d490698f7b854659a9851da2187414617cd5ed360af7efd63 \ - --hash=sha256:fe45f6870f7588ac7b2763ff1ce98cce59369717afe70cc353ec5218bc854bcc -zope.interface==5.1.0 \ - --hash=sha256:0103cba5ed09f27d2e3de7e48bb320338592e2fabc5ce1432cf33808eb2dfd8b \ - --hash=sha256:14415d6979356629f1c386c8c4249b4d0082f2ea7f75871ebad2e29584bd16c5 \ - --hash=sha256:1ae4693ccee94c6e0c88a4568fb3b34af8871c60f5ba30cf9f94977ed0e53ddd \ - --hash=sha256:1b87ed2dc05cb835138f6a6e3595593fea3564d712cb2eb2de963a41fd35758c \ - --hash=sha256:269b27f60bcf45438e8683269f8ecd1235fa13e5411de93dae3b9ee4fe7f7bc7 \ - --hash=sha256:27d287e61639d692563d9dab76bafe071fbeb26818dd6a32a0022f3f7ca884b5 \ - --hash=sha256:39106649c3082972106f930766ae23d1464a73b7d30b3698c986f74bf1256a34 \ - --hash=sha256:40e4c42bd27ed3c11b2c983fecfb03356fae1209de10686d03c02c8696a1d90e \ - --hash=sha256:461d4339b3b8f3335d7e2c90ce335eb275488c587b61aca4b305196dde2ff086 \ - --hash=sha256:4f98f70328bc788c86a6a1a8a14b0ea979f81ae6015dd6c72978f1feff70ecda \ - --hash=sha256:558a20a0845d1a5dc6ff87cd0f63d7dac982d7c3be05d2ffb6322a87c17fa286 \ - --hash=sha256:562dccd37acec149458c1791da459f130c6cf8902c94c93b8d47c6337b9fb826 \ - --hash=sha256:5e86c66a6dea8ab6152e83b0facc856dc4d435fe0f872f01d66ce0a2131b7f1d \ - --hash=sha256:60a207efcd8c11d6bbeb7862e33418fba4e4ad79846d88d160d7231fcb42a5ee \ - --hash=sha256:645a7092b77fdbc3f68d3cc98f9d3e71510e419f54019d6e282328c0dd140dcd \ - --hash=sha256:6874367586c020705a44eecdad5d6b587c64b892e34305bb6ed87c9bbe22a5e9 \ - --hash=sha256:74bf0a4f9091131de09286f9a605db449840e313753949fe07c8d0fe7659ad1e \ - --hash=sha256:7b726194f938791a6691c7592c8b9e805fc6d1b9632a833b9c0640828cd49cbc \ - --hash=sha256:8149ded7f90154fdc1a40e0c8975df58041a6f693b8f7edcd9348484e9dc17fe \ - --hash=sha256:8cccf7057c7d19064a9e27660f5aec4e5c4001ffcf653a47531bde19b5aa2a8a \ - --hash=sha256:911714b08b63d155f9c948da2b5534b223a1a4fc50bb67139ab68b277c938578 \ - --hash=sha256:a5f8f85986197d1dd6444763c4a15c991bfed86d835a1f6f7d476f7198d5f56a \ - --hash=sha256:a744132d0abaa854d1aad50ba9bc64e79c6f835b3e92521db4235a1991176813 \ - --hash=sha256:af2c14efc0bb0e91af63d00080ccc067866fb8cbbaca2b0438ab4105f5e0f08d \ - --hash=sha256:b054eb0a8aa712c8e9030065a59b5e6a5cf0746ecdb5f087cca5ec7685690c19 \ - --hash=sha256:b0becb75418f8a130e9d465e718316cd17c7a8acce6fe8fe07adc72762bee425 \ - --hash=sha256:b1d2ed1cbda2ae107283befd9284e650d840f8f7568cb9060b5466d25dc48975 \ - --hash=sha256:ba4261c8ad00b49d48bbb3b5af388bb7576edfc0ca50a49c11dcb77caa1d897e \ - --hash=sha256:d1fe9d7d09bb07228650903d6a9dc48ea649e3b8c69b1d263419cc722b3938e8 \ - --hash=sha256:d7804f6a71fc2dda888ef2de266727ec2f3915373d5a785ed4ddc603bbc91e08 \ - --hash=sha256:da2844fba024dd58eaa712561da47dcd1e7ad544a257482392472eae1c86d5e5 \ - --hash=sha256:dcefc97d1daf8d55199420e9162ab584ed0893a109f45e438b9794ced44c9fd0 \ - --hash=sha256:dd98c436a1fc56f48c70882cc243df89ad036210d871c7427dc164b31500dc11 \ - --hash=sha256:e74671e43ed4569fbd7989e5eecc7d06dc134b571872ab1d5a88f4a123814e9f \ - --hash=sha256:eb9b92f456ff3ec746cd4935b73c1117538d6124b8617bc0fe6fda0b3816e345 \ - --hash=sha256:ebb4e637a1fb861c34e48a00d03cffa9234f42bef923aec44e5625ffb9a8e8f9 \ - --hash=sha256:ef739fe89e7f43fb6494a43b1878a36273e5924869ba1d866f752c5812ae8d58 \ - --hash=sha256:f40db0e02a8157d2b90857c24d89b6310f9b6c3642369852cdc3b5ac49b92afc \ - --hash=sha256:f68bf937f113b88c866d090fea0bc52a098695173fc613b055a17ff0cf9683b6 \ - --hash=sha256:fb55c182a3f7b84c1a2d6de5fa7b1a05d4660d866b91dbf8d74549c57a1499e8 -zope.proxy==4.3.5 \ - --hash=sha256:00573dfa755d0703ab84bb23cb6ecf97bb683c34b340d4df76651f97b0bab068 \ - --hash=sha256:092049280f2848d2ba1b57b71fe04881762a220a97b65288bcb0968bb199ec30 \ - --hash=sha256:0cbd27b4d3718b5ec74fc65ffa53c78d34c65c6fd9411b8352d2a4f855220cf1 \ - --hash=sha256:17fc7e16d0c81f833a138818a30f366696653d521febc8e892858041c4d88785 \ - --hash=sha256:19577dfeb70e8a67249ba92c8ad20589a1a2d86a8d693647fa8385408a4c17b0 \ - --hash=sha256:207aa914576b1181597a1516e1b90599dc690c095343ae281b0772e44945e6a4 \ - --hash=sha256:219a7db5ed53e523eb4a4769f13105118b6d5b04ed169a283c9775af221e231f \ - --hash=sha256:2b50ea79849e46b5f4f2b0247a3687505d32d161eeb16a75f6f7e6cd81936e43 \ - --hash=sha256:5903d38362b6c716e66bbe470f190579c530a5baf03dbc8500e5c2357aa569a5 \ - --hash=sha256:5c24903675e271bd688c6e9e7df5775ac6b168feb87dbe0e4bcc90805f21b28f \ - --hash=sha256:5ef6bc5ed98139e084f4e91100f2b098a0cd3493d4e76f9d6b3f7b95d7ad0f06 \ - --hash=sha256:61b55ae3c23a126a788b33ffb18f37d6668e79a05e756588d9e4d4be7246ab1c \ - --hash=sha256:63ddb992931a5e616c87d3d89f5a58db086e617548005c7f9059fac68c03a5cc \ - --hash=sha256:6943da9c09870490dcfd50c4909c0cc19f434fa6948f61282dc9cb07bcf08160 \ - --hash=sha256:6ad40f85c1207803d581d5d75e9ea25327cd524925699a83dfc03bf8e4ba72b7 \ - --hash=sha256:6b44433a79bdd7af0e3337bd7bbcf53dd1f9b0fa66bf21bcb756060ce32a96c1 \ - --hash=sha256:6bbaa245015d933a4172395baad7874373f162955d73612f0b66b6c2c33b6366 \ - --hash=sha256:7007227f4ea85b40a2f5e5a244479f6a6dfcf906db9b55e812a814a8f0e2c28d \ - --hash=sha256:74884a0aec1f1609190ec8b34b5d58fb3b5353cf22b96161e13e0e835f13518f \ - --hash=sha256:7d25fe5571ddb16369054f54cdd883f23de9941476d97f2b92eb6d7d83afe22d \ - --hash=sha256:7e162bdc5e3baad26b2262240be7d2bab36991d85a6a556e48b9dfb402370261 \ - --hash=sha256:814d62678dc3a30f4aa081982d830b7c342cf230ffc9d030b020cb154eeebf9e \ - --hash=sha256:8878a34c5313ee52e20aa50b03138af8d472bae465710fb954d133a9bfd3c38d \ - --hash=sha256:a66a0d94e5b081d5d695e66d6667e91e74d79e273eee95c1747717ba9cb70792 \ - --hash=sha256:a69f5cbf4addcfdf03dda564a671040127a6b7c34cf9fe4973582e68441b63fa \ - --hash=sha256:b00f9f0c334d07709d3f73a7cb8ae63c6ca1a90c790a63b5e7effa666ef96021 \ - --hash=sha256:b6ed71e4a7b4690447b626f499d978aa13197a0e592950e5d7020308f6054698 \ - --hash=sha256:bdf5041e5851526e885af579d2f455348dba68d74f14a32781933569a327fddf \ - --hash=sha256:be034360dd34e62608419f86e799c97d389c10a0e677a25f236a971b2f40dac9 \ - --hash=sha256:cc8f590a5eed30b314ae6b0232d925519ade433f663de79cc3783e4b10d662ba \ - --hash=sha256:cd7a318a15fe6cc4584bf3c4426f092ed08c0fd012cf2a9173114234fe193e11 \ - --hash=sha256:cf19b5f63a59c20306e034e691402b02055c8f4e38bf6792c23cad489162a642 \ - --hash=sha256:cfc781ce442ec407c841e9aa51d0e1024f72b6ec34caa8fdb6ef9576d549acf2 \ - --hash=sha256:dea9f6f8633571e18bc20cad83603072e697103a567f4b0738d52dd0211b4527 \ - --hash=sha256:e4a86a1d5eb2cce83c5972b3930c7c1eac81ab3508464345e2b8e54f119d5505 \ - --hash=sha256:e7106374d4a74ed9ff00c46cc00f0a9f06a0775f8868e423f85d4464d2333679 \ - --hash=sha256:e98a8a585b5668aa9e34d10f7785abf9545fe72663b4bfc16c99a115185ae6a5 \ - --hash=sha256:f64840e68483316eb58d82c376ad3585ca995e69e33b230436de0cdddf7363f9 \ - --hash=sha256:f8f4b0a9e6683e43889852130595c8854d8ae237f2324a053cdd884de936aa9b \ - --hash=sha256:fc45a53219ed30a7f670a6d8c98527af0020e6fd4ee4c0a8fb59f147f06d816c diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py deleted file mode 100644 index 1515fe353..000000000 --- a/letsencrypt-auto-source/pieces/fetch.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Do downloading and JSON parsing without additional dependencies. :: - - # Print latest released version of LE to stdout: - python fetch.py --latest-version - - # Download letsencrypt-auto script from git tag v1.2.3 into the folder I'm - # in, and make sure its signature verifies: - python fetch.py --le-auto-script v1.2.3 - -On failure, return non-zero. - -""" - -from __future__ import print_function, unicode_literals - -from distutils.version import LooseVersion -from json import loads -from os import devnull, environ -from os.path import dirname, join -import re -import ssl -from subprocess import check_call, CalledProcessError -from sys import argv, exit -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler - from urllib2 import HTTPError, URLError -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler - from urllib.error import HTTPError, URLError - -PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq -OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 -xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp -9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij -n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH -cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ -CQIDAQAB ------END PUBLIC KEY----- -""") - -class ExpectedError(Exception): - """A novice-readable exception that also carries the original exception for - debugging""" - - -class HttpsGetter(object): - def __init__(self): - """Build an HTTPS opener.""" - # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. - if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=cert_none_context())) - else: - self._opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in self._opener.handlers: - if isinstance(handler, HTTPHandler): - self._opener.handlers.remove(handler) - - def get(self, url): - """Return the document contents pointed to by an HTTPS URL. - - If something goes wrong (404, timeout, etc.), raise ExpectedError. - - """ - try: - # socket module docs say default timeout is None: that is, no - # timeout - return self._opener.open(url, timeout=30).read() - except (HTTPError, IOError) as exc: - raise ExpectedError("Couldn't download %s." % url, exc) - - -def write(contents, dir, filename): - """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'wb') as file: - file.write(contents) - - -def latest_stable_version(get): - """Return the latest stable release of letsencrypt.""" - metadata = loads(get( - environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) - # metadata['info']['version'] actually returns the latest of any kind of - # release release, contrary to https://wiki.python.org/moin/PyPIJSON. - # The regex is a sufficient regex for picking out prereleases for most - # packages, LE included. - return str(max(LooseVersion(r) for r - in metadata['releases'].keys() - if re.match('^[0-9.]+$', r))) - - -def verified_new_le_auto(get, tag, temp_dir): - """Return the path to a verified, up-to-date letsencrypt-auto script. - - If the download's signature does not verify or something else goes wrong - with the verification process, raise ExpectedError. - - """ - le_auto_dir = environ.get( - 'LE_AUTO_DIR_TEMPLATE', - 'https://raw.githubusercontent.com/certbot/certbot/%s/' - 'letsencrypt-auto-source/') % tag - write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') - write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') - try: - with open(devnull, 'w') as dev_null: - check_call(['openssl', 'dgst', '-sha256', '-verify', - join(temp_dir, 'public_key.pem'), - '-signature', - join(temp_dir, 'letsencrypt-auto.sig'), - join(temp_dir, 'letsencrypt-auto')], - stdout=dev_null, - stderr=dev_null) - except CalledProcessError as exc: - raise ExpectedError("Couldn't verify signature of downloaded " - "certbot-auto.", exc) - - -def cert_none_context(): - """Create a SSLContext object to not check hostname.""" - # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.verify_mode = ssl.CERT_NONE - return context - - -def main(): - get = HttpsGetter().get - flag = argv[1] - try: - if flag == '--latest-version': - print(latest_stable_version(get)) - elif flag == '--le-auto-script': - tag = argv[2] - verified_new_le_auto(get, tag, dirname(argv[0])) - except ExpectedError as exc: - print(exc.args[0], exc.args[1]) - return 1 - else: - return 0 - - -if __name__ == '__main__': - exit(main()) diff --git a/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt deleted file mode 100644 index 8e745c9cd..000000000 --- a/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Contains the requirements for the letsencrypt package. -# -# Since the letsencrypt package depends on certbot and using pip with hashes -# requires that all installed packages have hashes listed, this allows -# dependency-requirements.txt to be used without requiring a hash for a -# (potentially unreleased) Certbot package. - -letsencrypt==0.7.0 \ - --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ - --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py deleted file mode 100755 index 7610c2686..000000000 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python -"""A small script that can act as a trust root for installing pip >=8 -Embed this in your project, and your VCS checkout is all you have to trust. In -a post-peep era, this lets you claw your way to a hash-checking version of pip, -with which you can install the rest of your dependencies safely. All it assumes -is Python 2.6 or better and *some* version of pip already installed. If -anything goes wrong, it will exit with a non-zero status code. -""" -# This is here so embedded copies are MIT-compliant: -# Copyright (c) 2016 Erik Rose -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -from __future__ import print_function -from distutils.version import StrictVersion -from hashlib import sha256 -from os import environ -from os.path import join -from shutil import rmtree -try: - from subprocess import check_output -except ImportError: - from subprocess import CalledProcessError, PIPE, Popen - - def check_output(*popenargs, **kwargs): - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be ' - 'overridden.') - process = Popen(stdout=PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd) - return output -import sys -from tempfile import mkdtemp -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse # 3.4 - - -__version__ = 1, 5, 1 -PIP_VERSION = '9.0.1' -DEFAULT_INDEX_BASE = 'https://pypi.python.org' - - -# wheel has a conditional dependency on argparse: -maybe_argparse = ( - [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' - 'argparse-1.4.0.tar.gz', - '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if sys.version_info < (2, 7, 0) else []) - - -# Be careful when updating the pinned versions here, in particular for pip. -# Indeed starting from 10.0, pip will build dependencies in isolation if the -# related projects are compliant with PEP 517. This is not something we want -# as of now, so the isolation build will need to be disabled wherever -# pipstrap is used (see https://github.com/certbot/certbot/issues/8256). -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz'.format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), - # This version of setuptools has only optional dependencies: - ('37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/' - 'setuptools-40.6.3.zip', - '3b474dad69c49f0d2d86696b68105f3a6f195f7ab655af12ef9a9c326d2b08f8'), - ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' - 'wheel-0.29.0.tar.gz', - '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') -] - - -class HashError(Exception): - def __str__(self): - url, path, actual, expected = self.args - return ('{url} did not match the expected hash {expected}. Instead, ' - 'it was {actual}. The file (left at {path}) may have been ' - 'tampered with.'.format(**locals())) - - -def hashed_download(url, temp, digest): - """Download ``url`` to ``temp``, make sure it has the SHA-256 ``digest``, - and return its path.""" - # Based on pip 1.4.1's URLOpener but with cert verification removed. Python - # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert - # authenticity has only privacy (not arbitrary code execution) - # implications, since we're checking hashes. - def opener(using_https=True): - opener = build_opener(HTTPSHandler()) - if using_https: - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) - return opener - - def read_chunks(response, chunk_size): - while True: - chunk = response.read(chunk_size) - if not chunk: - break - yield chunk - - parsed_url = urlparse(url) - response = opener(using_https=parsed_url.scheme == 'https').open(url) - path = join(temp, parsed_url.path.split('/')[-1]) - actual_hash = sha256() - with open(path, 'wb') as file: - for chunk in read_chunks(response, 4096): - file.write(chunk) - actual_hash.update(chunk) - - actual_digest = actual_hash.hexdigest() - if actual_digest != digest: - raise HashError(url, path, actual_digest, digest) - return path - - -def get_index_base(): - """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the - end if it's there; that is likely to give us the right dir. - """ - env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') - if env_var: - SIMPLE = '/simple' - if env_var.endswith(SIMPLE): - return env_var[:-len(SIMPLE)] - else: - return env_var - else: - return DEFAULT_INDEX_BASE - - -def main(): - python = sys.executable or 'python' - pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) - .decode('utf-8').split()[1]) - has_pip_cache = pip_version >= StrictVersion('6.0') - index_base = get_index_base() - temp = mkdtemp(prefix='pipstrap-') - try: - downloads = [hashed_download(index_base + '/packages/' + path, - temp, - digest) - for path, digest in PACKAGES] - # Calling pip as a module is the preferred way to avoid problems about pip self-upgrade. - command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] - # Disable cache since it is not used and it otherwise sometimes throws permission warnings: - command.extend(['--no-cache-dir'] if has_pip_cache else []) - command.extend(downloads) - check_output(command) - except HashError as exc: - print(exc) - except Exception: - rmtree(temp) - raise - else: - rmtree(temp) - return 0 - return 1 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/letsencrypt-auto-source/tests/signing.key b/letsencrypt-auto-source/tests/signing.key deleted file mode 100644 index b9964d00c..000000000 --- a/letsencrypt-auto-source/tests/signing.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEAsMoSzLYQ7E1sdSOkwelgtzKIh2qi3bpXuYtcfFC0XrvWig07 -1NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7GhFW0VdbxL6JdGzS2ShNWkX9hE9z+ -j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTTuUtJmmGcuk3a9Aq/sCT6DdfmTSdP -5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVglLsIVPBuy9IcgHidUQ96hJnoPsDCW -sHwX62495QKEarauyKQrJzFes0EY95orDM47Z5o/NDiQB11m91yNB0MmPYY9QSbn -OA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68iQIDAQABAoIBAQCJE3W2Mqk2f+XL -geKa1BjAkzcXQJCduYGRhUQlw/HGzoBPtGki56Tf53MeHTAkIGfIq3CAr1zRhiNv -8SQzvrLQIx/buvhxhcQJdzqsfwgNcqXT3/OliF34P3LMx8GUfPy/6xq2Qdv4fvwA -nLJH8wyDTKP6RxtdvUY7GSZ+Ln2QQv/3Nco7tax4GHNGom8iSgeH/YKTDnvitdqh -a0fr930QzU39TfOftLmasdmKUOIg8G2wr4Sy6Kn060+OUoQr1fZF5mnLvvQeILCK -uav91JkIeMLggzk+t88IJUFWdOoxv5hWTnNzHyt+/GYfovyRz2fKQMwzdh1F8iM5 -+867rEb9AoGBANn1ncemJBedDshStdCBUH0+2ExPrawveaXOZKnx8/VGFXNi0hAf -KzkntMWd5g5kB077FtKO9CYTBvK4pZBWIFLcJEqAz88JeXME6dfUbRucDr72ko+l -rcLHXj7F0IDVzj/9CphMGAhC9J/4YW9SPcSbMw6dQ6xOk73f1Vowve0DAoGBAM+k -/F+hVqCS3f22Bg9KuDtx+zCydaZxC842DgIkV1SO2iFhNHjnpQ5EIR0WrSYeV2n+ -rD7kVs5OH1HvnGScHaQKtAVqZClSwF14jzE+Aj8XDwxiHLSOhJgKlzfVX7h1ymMh -7fsslDl6xNGQ+40gubhkCLT5qABFKy1mrZ8b+3yDAoGAGLGUI6d2FVrM7vM3+Bx+ -gwIYvWSVl5l1XcypaPupmRNMoNsEU6FEY2BVQcJm6yB4F4GpD0f0709ejSdQUq7/ -UIPydKJtaNZ49QgMelBt4B/pJ8eFyVKLAjNWQSRmQAJ5MJS5m5Gbc2wqjOk2GMen -idvPiAtXPHFWmb9/S42UJwMCgYEAjymAe2qgcGtyNNfIC8kHhqzKdEPGi/ALJKzu -MZnewEURrcv4QpfrnA9rCUQ2Mz7eJA1bsqz6EJmaTIK4wEFGynA6uDUnQ7pzOL7D -cz7+i4MZc/89LVvJnY5Hvk4WBfboiDq/etq8g3jatGaSmTYD9la6DhTHORB3eYD+ -meHQHYMCgYEA18y9hnx2k4vNeBei4YXF4pAvKdwKLQD+CcP9ljb3VT+kXktjRA1C -aWj3HhMwvcxtttfkQzEnwwGRAkTEtNewJ8KFxhmc9nYElZTNZ+SuHD5Dkv8xqoj8 -NvG8rU1eiEyPwE2wQxpM5JLqbo7IWtR0dmptjKoF1gRxn6Wh4TwEiHA= ------END RSA PRIVATE KEY----- diff --git a/tests/letstest/README.md b/letstest/README.md similarity index 69% rename from tests/letstest/README.md rename to letstest/README.md index 76db57153..c569d1e8f 100644 --- a/tests/letstest/README.md +++ b/letstest/README.md @@ -14,17 +14,13 @@ Simple AWS testfarm scripts for certbot client testing are needed, they need to be requested via online webform. ## Installation and configuration -These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you're -on a Debian based system, make sure you also have the python3-venv package -installed. If you have Python 3 installed, you can use requirements.txt to -create a virtual environment with a known set of dependencies by running: -``` -python3 -m venv venv3 -. ./venv3/bin/activate -pip install --requirement requirements.txt -``` -You can then configure AWS credentials and create a key by running: +This package is installed in the Certbot development environment that is +created by following the instructions at +https://certbot.eff.org/docs/contributing.html#running-a-local-copy-of-the-client. + +After activating that virtual environment, you can then configure AWS +credentials and create a key by running: ``` >aws configure --profile [interactive: enter secrets for IAM role] @@ -35,9 +31,9 @@ Note: whatever you pick for `` will be shown to other users with AWS a When prompted for a default region name, enter: `us-east-1`. ## Usage -To run tests, activate the virtual environment you created above and run: +To run tests, activate the virtual environment you created above and from this directory run: ``` ->python multitester.py targets.yaml /path/to/your/key.pem scripts/ +>letstest targets/targets.yaml /path/to/your/key.pem scripts/ ``` You can only run up to two tests at once. The following error is often indicative of there being too many AWS instances running on our account: @@ -52,15 +48,14 @@ aws ec2 terminate-instances --profile --instance-ids $(aws ec2 de It will take a minute for these instances to shut down and become available again. Running this will invalidate any in progress tests. -A folder named `letest-` is also created with a log file from each instance of the test and a file named "results" containing the output above. +A temporary directory whose name is output by the tests is also created with a log file from each instance of the test and a file named "results" containing the output above. The tests take quite a while to run. ## Scripts Example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed to them at runtime via environment variables. test_apache2.sh is a useful reference. -Note that the 
test_letsencrypt_auto_*
scripts pull code from PyPI using the letsencrypt-auto script, -__not__ the local python code. test_apache2 runs the dev venv and does local tests. +test_apache2 runs the dev venv and does local tests. See: - https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html diff --git a/letstest/letstest/__init__.py b/letstest/letstest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/letstest/multitester.py b/letstest/letstest/multitester.py similarity index 97% rename from tests/letstest/multitester.py rename to letstest/letstest/multitester.py index 2e4399000..a56bf0f37 100644 --- a/tests/letstest/multitester.py +++ b/letstest/letstest/multitester.py @@ -22,7 +22,7 @@ Usage: >aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \ --query 'KeyMaterial' --output text > MyKeyPair.pem then: ->python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_leauto_upgrades.sh +>letstest targets/targets.yaml MyKeyPair.pem HappyHacker scripts/test_sdists.sh see: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html @@ -33,6 +33,7 @@ from multiprocessing import Manager import os import socket import sys +import tempfile import time import traceback import urllib.error as urllib_error @@ -55,7 +56,7 @@ parser.add_argument('key_file', parser.add_argument('aws_profile', help='profile for AWS (i.e. as in ~/.aws/certificates)') parser.add_argument('test_script', - default='test_letsencrypt_auto_certonly_standalone.sh', + default='test_sdists.sh', help='path of bash script in to deploy and run') parser.add_argument('--repo', default='https://github.com/letsencrypt/letsencrypt.git', @@ -374,9 +375,8 @@ def main(): # Set up local copy of git repo #------------------------------------------------------------------------------- - log_dir = "letest-%d"%int(time.time()) #points to logging / working directory - print("Making local dir for test repo and logs: %s"%log_dir) - local_cxn.local('mkdir %s'%log_dir) + log_dir = tempfile.mkdtemp() # points to logging / working directory + print("Local dir for test repo and logs: %s"%log_dir) try: # figure out what git object to test and locally create it in log_dir @@ -496,12 +496,17 @@ def main(): outputs = [outq for outq in iter(outqueue.get, SENTINEL)] outputs.sort(key=lambda x: x[0]) failed = False + results_msg = "" for outq in outputs: ii, target, status = outq if status == Status.FAIL: failed = True - print('%d %s %s'%(ii, target['name'], status)) + with open(log_dir+'/'+'%d_%s.log'%(ii,target['name']), 'r') as f: + print(target['name'] + " test failed. Test log:") + print(f.read()) + results_msg = results_msg + '%d %s %s\n'%(ii, target['name'], status) results_file.write('%d %s %s\n'%(ii, target['name'], status)) + print(results_msg) if len(outputs) != num_processes: failed = True failure_message = 'FAILURE: Some target machines failed to run and were not tested. ' +\ diff --git a/tests/letstest/scripts/bootstrap_os_packages.sh b/letstest/scripts/bootstrap_os_packages.sh similarity index 76% rename from tests/letstest/scripts/bootstrap_os_packages.sh rename to letstest/scripts/bootstrap_os_packages.sh index 7ad93f63e..3f4c6e30e 100755 --- a/tests/letstest/scripts/bootstrap_os_packages.sh +++ b/letstest/scripts/bootstrap_os_packages.sh @@ -34,9 +34,9 @@ DeterminePythonVersion() { } BootstrapDebCommon() { - apt-get update || error apt-get update hit problems but continuing anyway... + sudo apt-get update || error apt-get update hit problems but continuing anyway... - apt-get install -y --no-install-recommends \ + sudo apt-get install -y --no-install-recommends \ python3 \ python3-dev \ python3-venv \ @@ -46,8 +46,19 @@ BootstrapDebCommon() { openssl \ libffi-dev \ ca-certificates \ + build-essential \ + curl \ make # needed on debian 9 arm64 which doesn't have a python3 pynacl wheel + # make sure rust isn't installed by the package manager + if ! sudo apt-get remove -y rustc; then + error "Could not remove existing rust. Aborting bootstrap!" + exit 1 + fi + + # Install rust for cryptography (needed on Debian) + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + . $HOME/.cargo/env } # Sets TOOL to the name of the package manager @@ -79,6 +90,7 @@ BootstrapRpmCommonBase() { libffi-devel redhat-rpm-config ca-certificates + cargo " # Add the python packages @@ -92,7 +104,7 @@ BootstrapRpmCommonBase() { " fi - if ! $TOOL install -y $pkgs; then + if ! sudo $TOOL install -y $pkgs; then error "Could not install OS dependencies. Aborting bootstrap!" exit 1 fi @@ -105,8 +117,8 @@ BootstrapRpmPython3() { python3-devel " - if ! $TOOL list 'python3*-devel' >/dev/null 2>&1; then - yum-config-manager --enable rhui-REGION-rhel-server-extras rhui-REGION-rhel-server-optional + if ! sudo $TOOL list 'python3*-devel' >/dev/null 2>&1; then + sudo yum-config-manager --enable rhui-REGION-rhel-server-extras rhui-REGION-rhel-server-optional fi BootstrapRpmCommonBase "$python_pkgs" diff --git a/tests/letstest/scripts/test_apache2.sh b/letstest/scripts/test_apache2.sh similarity index 77% rename from tests/letstest/scripts/test_apache2.sh rename to letstest/scripts/test_apache2.sh index 77dc35f1e..830ae44b2 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/letstest/scripts/test_apache2.sh @@ -12,25 +12,6 @@ then # For apache 2.4, set up ServerName sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' $CONFFILE - if [ $(python3 -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -lt 36 ] - then - # Upgrade python version using pyenv because py3.5 is deprecated - # Don't upgrade if it's already 3.8 because pyenv doesn't work great on arm, and - # our arm representative happens to be ubuntu20, which already has a perfectly - # good version of python. - sudo apt-get install -y make gcc build-essential libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev python-openssl git # pyenv deps - curl https://pyenv.run | bash - export PATH="~/.pyenv/bin:$PATH" - pyenv init - - pyenv virtualenv-init - - pyenv install 3.8.5 - pyenv global 3.8.5 - # you do, in fact need to run these again, exactly like this. - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - fi elif [ "$OS_TYPE" = "centos" ] then CONFFILE=/etc/httpd/conf/httpd.conf @@ -59,7 +40,7 @@ fi cd letsencrypt echo "Bootstrapping dependencies..." -sudo tests/letstest/scripts/bootstrap_os_packages.sh +sudo letstest/scripts/bootstrap_os_packages.sh if [ $? -ne 0 ] ; then exit 1 fi @@ -113,7 +94,7 @@ elif [ "$OS_TYPE" = "centos" ]; then fi OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9) APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8) -"venv/bin/python" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" +"venv/bin/python" letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" if [ $? -ne 0 ] ; then FAIL=1 fi diff --git a/tests/letstest/scripts/test_openssl_version.py b/letstest/scripts/test_openssl_version.py similarity index 100% rename from tests/letstest/scripts/test_openssl_version.py rename to letstest/scripts/test_openssl_version.py diff --git a/letstest/scripts/test_sdists.sh b/letstest/scripts/test_sdists.sh new file mode 100755 index 000000000..562169524 --- /dev/null +++ b/letstest/scripts/test_sdists.sh @@ -0,0 +1,53 @@ +#!/bin/sh -xe + +cd letsencrypt + +BOOTSTRAP_SCRIPT="letstest/scripts/bootstrap_os_packages.sh" +VENV_PATH=venv + +# install OS packages +. $BOOTSTRAP_SCRIPT + +# setup venv +python3 -m venv $VENV_PATH +$VENV_PATH/bin/python3 tools/pipstrap.py +. "$VENV_PATH/bin/activate" +# pytest is needed to run tests on our packages so we install a pinned version here. +tools/pip_install.py pytest + +# setup constraints +TEMP_DIR=$(mktemp -d) +CONSTRAINTS="$TEMP_DIR/constraints.txt" +cp tools/requirements.txt "$CONSTRAINTS" + +# We pin cryptography to 3.1.1 and pyopenssl to 19.1.0 specifically for CentOS 7 / RHEL 7 +# because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been +# dropped on cryptography>=3.2 and pyopenssl>=20.0.0. +# Using this old version of OpenSSL would break the cryptography and pyopenssl wheels builds. +if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then + sed -i 's|cryptography==.*|cryptography==3.1.1|g' "$CONSTRAINTS" + sed -i 's|pyopenssl==.*|pyopenssl==19.1.0|g' "$CONSTRAINTS" +fi + + +PLUGINS="certbot-apache certbot-nginx" +# build sdists +for pkg_dir in acme certbot $PLUGINS; do + cd $pkg_dir + python setup.py clean + rm -rf build dist + python setup.py sdist + mv dist/* $TEMP_DIR + cd - +done + +VERSION=$(python letstest/scripts/version.py) +# test sdists +cd $TEMP_DIR +for pkg in acme certbot $PLUGINS; do + tar -xvf "$pkg-$VERSION.tar.gz" + cd "$pkg-$VERSION" + PIP_CONSTRAINT=../constraints.txt PIP_NO_BINARY=:all: pip install . + python -m pytest + cd - +done diff --git a/letsencrypt-auto-source/version.py b/letstest/scripts/version.py similarity index 75% rename from letsencrypt-auto-source/version.py rename to letstest/scripts/version.py index d70ffefac..6e538b032 100755 --- a/letsencrypt-auto-source/version.py +++ b/letstest/scripts/version.py @@ -1,8 +1,7 @@ #!/usr/bin/env python """Get the current Certbot version number. -Provides simple utilities for determining the Certbot version number and -building letsencrypt-auto. +Provides a simple utility for determining the Certbot version number """ from __future__ import print_function @@ -10,10 +9,10 @@ from os.path import abspath, dirname, join import re -def certbot_version(build_script_dir): +def certbot_version(letstest_scripts_dir): """Return the version number stamped in certbot/__init__.py.""" return re.search('''^__version__ = ['"](.+)['"].*''', - file_contents(join(dirname(build_script_dir), + file_contents(join(dirname(dirname(letstest_scripts_dir)), 'certbot', 'certbot', '__init__.py')), diff --git a/letstest/setup.py b/letstest/setup.py new file mode 100644 index 000000000..a552cf920 --- /dev/null +++ b/letstest/setup.py @@ -0,0 +1,45 @@ +from setuptools import find_packages +from setuptools import setup + +setup( + name='letstest', + version='1.0', + description='Test Certbot on different AWS images', + url='https://github.com/certbot/certbot', + author='Certbot Project', + author_email='certbot-dev@eff.org', + license='Apache License 2.0', + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=[ + # awscli isn't required by the tests themselves, but it is a useful + # tool to have when using these tests to generate keys and control + # running instances so the dependency is declared here for convenience. + 'awscli', + 'boto3', + 'botocore', + # The API from Fabric 2.0+ is used instead of the 1.0 API. + 'fabric>=2', + 'pyyaml', + ], + entry_points={ + 'console_scripts': [ + 'letstest=letstest.multitester:main', + ], + } +) diff --git a/tests/letstest/apache2_targets.yaml b/letstest/targets/apache2_targets.yaml similarity index 82% rename from tests/letstest/apache2_targets.yaml rename to letstest/targets/apache2_targets.yaml index 2663782ce..c0c08be12 100644 --- a/tests/letstest/apache2_targets.yaml +++ b/letstest/targets/apache2_targets.yaml @@ -21,11 +21,6 @@ targets: type: ubuntu virt: hvm user: ubuntu - - ami: ami-09677e0a6b14905b0 - name: ubuntu16.04LTS - type: ubuntu - virt: hvm - user: ubuntu #----------------------------------------------------------------------------- # Debian - ami: ami-01db78123b2b99496 @@ -33,11 +28,6 @@ targets: type: ubuntu virt: hvm user: admin - - ami: ami-003f19e0e687de1cd - name: debian9 - type: ubuntu - virt: hvm - user: admin #----------------------------------------------------------------------------- # CentOS - ami: ami-9887c6e7 diff --git a/tests/letstest/targets.yaml b/letstest/targets/targets.yaml similarity index 100% rename from tests/letstest/targets.yaml rename to letstest/targets/targets.yaml diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index d53fba88b..b3f1349e0 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -70,7 +70,15 @@ parts: - python3-pkg-resources - python3.8-minimal # To build cryptography and cffi if needed - build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev] + build-packages: + - gcc + - git + - libaugeas-dev + - build-essential + - libssl-dev + - libffi-dev + - python3-dev + - cargo build-environment: - SNAPCRAFT_PYTHON_VENV_ARGS: --upgrade # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the @@ -85,9 +93,7 @@ parts: snapcraftctl build override-pull: | snapcraftctl pull - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_constraints.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" - echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + grep -v python-augeas "${SNAPCRAFT_PART_SRC}/tools/requirements.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` shared-metadata: plugin: dump diff --git a/tests/letstest/auto_targets.yaml b/tests/letstest/auto_targets.yaml deleted file mode 100644 index 01d410227..000000000 --- a/tests/letstest/auto_targets.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# These images are located in us-east-1. - -targets: - #----------------------------------------------------------------------------- - #Ubuntu - - ami: ami-095192256fe1477ad - name: ubuntu18.04LTS - type: ubuntu - virt: hvm - user: ubuntu - - ami: ami-09677e0a6b14905b0 - name: ubuntu16.04LTS - type: ubuntu - virt: hvm - user: ubuntu - #----------------------------------------------------------------------------- - # Debian - - ami: ami-01db78123b2b99496 - name: debian10 - type: ubuntu - virt: hvm - user: admin - - ami: ami-003f19e0e687de1cd - name: debian9 - type: ubuntu - virt: hvm - user: admin - - ami: ami-0ed54dd1b25657636 - name: debian9_arm64 - type: ubuntu - virt: hvm - user: admin - machine_type: a1.medium - #----------------------------------------------------------------------------- - # Other Redhat Distros - - ami: ami-0916c408cb02e310b - name: RHEL7 - type: centos - virt: hvm - user: ec2-user - - ami: ami-0c322300a1dd5dc79 - name: RHEL8 - type: centos - virt: hvm - user: ec2-user - #----------------------------------------------------------------------------- - # CentOS - # These Marketplace AMIs must, irritatingly, have their terms manually - # agreed to on the AWS marketplace site for any new AWS account using them... - - ami: ami-9887c6e7 - name: centos7 - type: centos - virt: hvm - user: centos - - ami: ami-01ca03df4a6012157 - name: centos8 - type: centos - virt: hvm - user: centos diff --git a/tests/letstest/requirements.txt b/tests/letstest/requirements.txt deleted file mode 100644 index d30c507cf..000000000 --- a/tests/letstest/requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -awscli==1.18.88 -bcrypt==3.1.7 -boto3==1.14.11 -botocore==1.17.11 -cffi==1.14.0 -colorama==0.4.3 -cryptography==2.8 -docutils==0.15.2 -enum34==1.1.9 -fabric==2.5.0 -invoke==1.4.1 -ipaddress==1.0.23 -jmespath==0.9.5 -paramiko==2.7.1 -pyasn1==0.4.8 -pycparser==2.19 -PyNaCl==1.3.0 -python-dateutil==2.8.1 -PyYAML==5.3 -rsa==3.4.2 -s3transfer==0.3.3 -six==1.14.0 -urllib3==1.25.8 diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh deleted file mode 100755 index 3964364e1..000000000 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/bash -xe -set -o pipefail - -# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME -# are dynamically set at execution - -cd letsencrypt - -if ! command -v git ; then - if [ "$OS_TYPE" = "ubuntu" ] ; then - sudo apt-get update - fi - if ! ( sudo apt-get install -y git || sudo yum install -y git-all || sudo yum install -y git || sudo dnf install -y git ) ; then - echo git installation failed! - exit 1 - fi -fi -# If we're on a RHEL 6 based system, we can be confident Python is already -# installed because the package manager is written in Python. -if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then - # 0.20.0 is the latest version of letsencrypt-auto that doesn't install - # Python 3 on RHEL 6. - INITIAL_VERSION="0.20.0" - RUN_RHEL6_TESTS=1 -else - # 0.39.0 is the oldest version of letsencrypt-auto that works on CentOS 8. - INITIAL_VERSION="0.39.0" -fi - -git checkout -f "v$INITIAL_VERSION" letsencrypt-auto -if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 | grep "^certbot $INITIAL_VERSION$" ; then - echo initial installation appeared to fail - exit 1 -fi - -if command -v python; then - PYTHON_NAME="python" -else - PYTHON_NAME="python3" -fi - -# Now that python and openssl have been installed, we can set up a fake server -# to provide a new version of letsencrypt-auto. First, we start the server and -# directory to be served. -MY_TEMP_DIR=$(mktemp -d) -PORT_FILE="$MY_TEMP_DIR/port" -LOG_FILE="$MY_TEMP_DIR/log" -SERVER_PATH=$("$PYTHON_NAME" tools/readlink.py tools/simple_http_server.py) -cd "$MY_TEMP_DIR" -# We set PYTHONUNBUFFERED to disable buffering of output to LOG_FILE -PYTHONUNBUFFERED=1 "$PYTHON_NAME" "$SERVER_PATH" 0 > $PORT_FILE 2> "$LOG_FILE" & -SERVER_PID=$! -trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT -cd ~- - -# Then, we set up the files to be served. -FAKE_VERSION_NUM="99.99.99" -echo "{\"releases\": {\"$FAKE_VERSION_NUM\": null}}" > "$MY_TEMP_DIR/json" -LE_AUTO_SOURCE_DIR="$MY_TEMP_DIR/v$FAKE_VERSION_NUM" -NEW_LE_AUTO_PATH="$LE_AUTO_SOURCE_DIR/letsencrypt-auto" -mkdir "$LE_AUTO_SOURCE_DIR" -cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_SOURCE_DIR/letsencrypt-auto" -SIGNING_KEY="letsencrypt-auto-source/tests/signing.key" -openssl dgst -sha256 -sign "$SIGNING_KEY" -out "$NEW_LE_AUTO_PATH.sig" "$NEW_LE_AUTO_PATH" - -# Next, we wait for the server to start and get the port number. -sleep 5s -SERVER_PORT=$(sed -n 's/.*port \([0-9]\+\).*/\1/p' "$PORT_FILE") - -# Finally, we set the necessary certbot-auto environment variables. -export LE_AUTO_DIR_TEMPLATE="http://localhost:$SERVER_PORT/%s/" -export LE_AUTO_JSON_URL="http://localhost:$SERVER_PORT/json" -export LE_AUTO_PUBLIC_KEY="-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg -tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G -hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT -uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl -LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 -Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 -iQIDAQAB ------END PUBLIC KEY----- -" - -if [ "$RUN_RHEL6_TESTS" = 1 ]; then - if command -v python3; then - echo "Didn't expect Python 3 to be installed!" - exit 1 - fi - cp letsencrypt-auto cb-auto - if ! ./cb-auto -v --debug --version 2>&1 | grep "$INITIAL_VERSION" ; then - echo "Certbot shouldn't have updated to a new version!" - exit 1 - fi - # Create a 2nd venv at the old path to ensure we properly handle the (unlikely) case of two separate virtual environments below. - HOME=${HOME:-~root} - XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} - OLD_VENV_PATH="$XDG_DATA_HOME/letsencrypt" - export VENV_PATH="$OLD_VENV_PATH" - if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 | grep "^certbot $INITIAL_VERSION$" ; then - echo second installation appeared to fail - exit 1 - fi - unset VENV_PATH -fi - -if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then - echo "Had problems checking for updates!" - exit 1 -fi - -# Since certbot-auto is deprecated, we expect it to leave existing Certbot -# installations unmodified so we check for the same version that was initially -# installed below. -EXPECTED_VERSION="$INITIAL_VERSION" - -if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then - echo unexpected certbot version found - exit 1 -fi - -if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then - echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ - exit 1 -fi - -# Now let's test if letsencrypt-auto still tries to upgrade to a new version. -# Regardless of the OS, versions of the script with development version numbers -# ending in .dev0 will not upgrade. See -# https://github.com/certbot/certbot/blob/bdfb9f19c4086a60ef010d2431768850c26d838a/certbot-auto#L1947-L1948. -# In order to test the process of different OSes setting NO_SELF_UPGRADE as -# part of the script's deprecation, we make use of the fact that -# letsencrypt-auto should still attempt to fetch the version number from PyPI -# even if it has a development version number unless NO_SELF_UPGRADE is set in -# which case all of that logic should be skipped. -# -# First we make a copy of the current server logs. -PREVIOUS_LOG_FILE="$MY_TEMP_DIR/previous-log" -cp "$LOG_FILE" "$PREVIOUS_LOG_FILE" - -# Next we run letsencrypt-auto and make sure there were no problems checking -# for updates, the Certbot install still works, the version number is what -# we expect, and it prints a message about not receiving updates. -if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then - echo "Had problems checking for updates!" - exit 1 -fi -if ! ./letsencrypt-auto -v --debug --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then - echo unexpected certbot version found - exit 1 -fi -if ! ./letsencrypt-auto -v --debug --version 2>&1 | grep "will no longer receive updates" ; then - echo script did not print warning about not receiving updates! - exit 1 -fi - -# Finally, we check if our local server received more requests. -if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then - echo our local server received unexpected requests - exit 1 -fi diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh deleted file mode 100755 index 9573ab690..000000000 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -x -set -eo pipefail - -# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME are dynamically set at execution - -# with curl, instance metadata available from EC2 metadata service: -#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) -#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) -#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) - -cd letsencrypt -LE_AUTO_DIR="/usr/local/bin" -LE_AUTO_PATH="$LE_AUTO_DIR/letsencrypt-auto" -sudo cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_PATH" -sudo chown root "$LE_AUTO_PATH" -sudo chmod 0755 "$LE_AUTO_PATH" -export PATH="$LE_AUTO_DIR:$PATH" - -# Since certbot-auto is deprecated, we expect certbot-auto to error and -# refuse to install Certbot. -set +o pipefail -if ! letsencrypt-auto --debug --version | grep "Certbot cannot be installed."; then - echo "letsencrypt-auto didn't report being uninstallable." - exit 1 -fi -if [ ${PIPESTATUS[0]} != 1 ]; then - echo "letsencrypt-auto didn't exit with status 1 as expected" - exit 1 -fi diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh deleted file mode 100755 index becdd6d9a..000000000 --- a/tests/letstest/scripts/test_sdists.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh -xe - -cd letsencrypt - -BOOTSTRAP_SCRIPT="tests/letstest/scripts/bootstrap_os_packages.sh" -VENV_PATH=venv - -# install OS packages -sudo $BOOTSTRAP_SCRIPT - -# setup venv -# We strip the hashes because the venv creation script includes unhashed -# constraints in the commands given to pip and the mix of hashed and unhashed -# packages makes pip error out. -python3 tools/strip_hashes.py tools/pipstrap_constraints.txt > constraints.txt -python3 tools/strip_hashes.py tools/certbot_constraints.txt > requirements.txt - -# We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7 -# because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been -# dropped on cryptography>=3.2 and pyOpenSSL>=20.0.0. -# Using this old version of OpenSSL would break the cryptography and pyOpenSSL wheels builds. -if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then - sed -i 's|cryptography==.*|cryptography==3.1.1|g' requirements.txt - sed -i 's|pyOpenSSL==.*|pyOpenSSL==19.1.0|g' requirements.txt -fi - -python3 -m venv $VENV_PATH -$VENV_PATH/bin/python3 tools/pipstrap.py -PIP_CONSTRAINT=constraints.txt PIP_NO_BINARY=:all: $VENV_PATH/bin/python3 -m pip install --requirement requirements.txt -. "$VENV_PATH/bin/activate" -# pytest is needed to run tests on some of our packages so we install a pinned version here. -tools/pip_install.py pytest - -PLUGINS="certbot-apache certbot-nginx" -TEMP_DIR=$(mktemp -d) - -# build sdists -for pkg_dir in acme certbot $PLUGINS; do - cd $pkg_dir - python setup.py clean - rm -rf build dist - python setup.py sdist - mv dist/* $TEMP_DIR - cd - -done - -VERSION=$(python letsencrypt-auto-source/version.py) -# test sdists -cd $TEMP_DIR -for pkg in acme certbot $PLUGINS; do - tar -xvf "$pkg-$VERSION.tar.gz" - cd "$pkg-$VERSION" - python setup.py build - python -m pytest - python setup.py install - cd - -done diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh deleted file mode 100755 index 858fc1f18..000000000 --- a/tests/letstest/scripts/test_tests.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -xe -# -# This script is useful for testing that the packages we've built for a release -# work on a variety of systems. For an example of the kinds of problems that -# can occur, see https://github.com/certbot/certbot/issues/3455. - -REPO_ROOT="letsencrypt" -LE_AUTO="$REPO_ROOT/letsencrypt-auto-source/letsencrypt-auto" -LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" -MODULES="acme certbot certbot-apache certbot-nginx" -PIP_INSTALL="tools/pip_install.py" -VENV_NAME=venv -BOOTSTRAP_SCRIPT="$REPO_ROOT/tests/letstest/scripts/bootstrap_os_packages.sh" -VENV_SCRIPT="tools/venv.py" - -sudo $BOOTSTRAP_SCRIPT - -cd $REPO_ROOT -$VENV_SCRIPT -. $VENV_NAME/bin/activate -"$PIP_INSTALL" pytest - -# To run tests that aren't packaged in modules, run pytest -# from the repo root. The directory structure should still -# cause the installed packages to be tested while using -# the tests available in the subdirectories. - -for module in $MODULES ; do - echo testing $module - pytest -v $module -done diff --git a/tests/modification-check.py b/tests/modification-check.py index 357b25350..8f3ae1264 100755 --- a/tests/modification-check.py +++ b/tests/modification-check.py @@ -1,122 +1,58 @@ #!/usr/bin/env python +"""Ensures there have been no changes to important certbot-auto files.""" -from __future__ import print_function - +import hashlib import os -import shutil -import subprocess -import sys -import tempfile -from urllib.request import urlretrieve -def find_repo_path(): +# Relative to the root of the Certbot repo, these files are expected to exist +# and have the SHA-256 hashes contained in this dictionary. These hashes were +# taken from our v1.14.0 tag which was the last release we intended to make +# changes to certbot-auto. +# +# certbot-auto, letsencrypt-auto, and letsencrypt-auto-source/certbot-auto.asc +# can be removed from this dict after coordinating with tech ops to ensure we +# get the behavior we want from https://dl.eff.org. See +# https://github.com/certbot/certbot/issues/8742 for more info. +# +# Deleting letsencrypt-auto-source/letsencrypt-auto and +# letsencrypt-auto-source/letsencrypt-auto.sig can be done once we're +# comfortable breaking any certbot-auto scripts that haven't already updated to +# the last version. See +# https://opensource.eff.org/eff-open-source/pl/65geri7c4tr6iqunc1rpb3mpna for +# more info. +EXPECTED_FILES = { + 'certbot-auto': + 'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2', + 'letsencrypt-auto': + 'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2', + os.path.join('letsencrypt-auto-source', 'letsencrypt-auto'): + 'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2', + os.path.join('letsencrypt-auto-source', 'certbot-auto.asc'): + '0558ba7bd816732b38c092e8fedb6033dad01f263e290ec6b946263aaf6625a8', + os.path.join('letsencrypt-auto-source', 'letsencrypt-auto.sig'): + '61c036aabf75da350b0633da1b2bef0260303921ecda993455ea5e6d3af3b2fe', +} + + +def find_repo_root(): return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -# We do not use filecmp.cmp to take advantage of universal newlines -# handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. -# As a consequence, this function will not work correctly if executed by Python 2.x on Windows. -# But it will work correctly on Linux for any version, because every file tested will be LF. -def compare_files(path_1, path_2): - l1 = l2 = True - with open(path_1, 'r') as f1, open(path_2, 'r') as f2: - line = 1 - while l1 and l2: - line += 1 - l1 = f1.readline() - l2 = f2.readline() - if l1 != l2: - print('---') - print(( - 'While comparing {0} (1) and {1} (2), a difference was found at line {2}:' - .format(os.path.basename(path_1), os.path.basename(path_2), line))) - print('(1): {0}'.format(repr(l1))) - print('(2): {0}'.format(repr(l2))) - print('---') - return False - return True +def sha256_hash(filename): + hash_object = hashlib.sha256() + with open(filename, 'rb') as f: + hash_object.update(f.read()) + return hash_object.hexdigest() -def validate_scripts_content(repo_path, temp_cwd): - errors = False - - if not compare_files( - os.path.join(repo_path, 'certbot-auto'), - os.path.join(repo_path, 'letsencrypt-auto')): - print('Root certbot-auto and letsencrypt-auto differ.') - errors = True - else: - shutil.copyfile( - os.path.join(repo_path, 'certbot-auto'), - os.path.join(temp_cwd, 'local-auto')) - shutil.copy(os.path.normpath(os.path.join( - repo_path, - 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) - - # Compare file against current version in the target branch - branch = os.environ.get('TARGET_BRANCH', 'master') - url = ( - 'https://raw.githubusercontent.com/certbot/certbot/{0}/certbot-auto' - .format(branch)) - urlretrieve(url, os.path.join(temp_cwd, 'certbot-auto')) - - if compare_files( - os.path.join(temp_cwd, 'certbot-auto'), - os.path.join(temp_cwd, 'local-auto')): - print('Root *-auto were unchanged') - else: - # Compare file against the latest released version - latest_version = subprocess.check_output( - [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) - subprocess.check_call( - [sys.executable, 'fetch.py', '--le-auto-script', - 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) - if compare_files( - os.path.join(temp_cwd, 'letsencrypt-auto'), - os.path.join(temp_cwd, 'local-auto')): - print('Root *-auto were updated to the latest version.') - else: - print('Root *-auto have unexpected changes.') - errors = True - - return errors def main(): - repo_path = find_repo_path() - temp_cwd = tempfile.mkdtemp() - errors = False + repo_root = find_repo_root() + for filename, expected_hash in EXPECTED_FILES.items(): + filepath = os.path.join(repo_root, filename) + assert sha256_hash(filepath) == expected_hash, f'unexpected changes to {filepath}' + print('All certbot-auto files have correct hashes.') - try: - errors = validate_scripts_content(repo_path, temp_cwd) - - shutil.copyfile( - os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), - os.path.join(temp_cwd, 'original-lea') - ) - subprocess.check_call([sys.executable, os.path.normpath(os.path.join( - repo_path, 'letsencrypt-auto-source/build.py'))]) - shutil.copyfile( - os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), - os.path.join(temp_cwd, 'build-lea') - ) - shutil.copyfile( - os.path.join(temp_cwd, 'original-lea'), - os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')) - ) - - if not compare_files( - os.path.join(temp_cwd, 'original-lea'), - os.path.join(temp_cwd, 'build-lea')): - print('Script letsencrypt-auto-source/letsencrypt-auto ' - 'doesn\'t match output of build.py.') - errors = True - else: - print('Script letsencrypt-auto-source/letsencrypt-auto matches output of build.py.') - finally: - shutil.rmtree(temp_cwd) - - return errors if __name__ == '__main__': - if main(): - sys.exit(1) + main() diff --git a/tools/_release.sh b/tools/_release.sh index e9e0e49f0..7dd4601ff 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -30,9 +30,6 @@ echo Releasing production version "$version"... nextversion="$2" RELEASE_BRANCH="candidate-$version" -if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then - RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" -fi RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} # Needed to fix problems with git signatures and pinentry export GPG_TTY=$(tty) @@ -40,14 +37,13 @@ export GPG_TTY=$(tty) # port for a local Python Package Index (used in testing) PORT=${PORT:-1234} -# subpackages to be released (the way developers think about them) -SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" - # subpackages to be released (the way the script thinks about them) -SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" -SUBPKGS_NO_CERTBOT="$SUBPKGS_IN_AUTO_NO_CERTBOT $SUBPKGS_NOT_IN_AUTO" -SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" +SUBPKGS_NO_CERTBOT="acme certbot-apache certbot-nginx certbot-dns-cloudflare certbot-dns-cloudxns \ + certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy \ + certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns \ + certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 \ + certbot-dns-sakuracloud" +SUBPKGS="certbot $SUBPKGS_NO_CERTBOT" # certbot_compatibility_test is not packaged because: # - it is not meant to be used by anyone else than Certbot devs # - it causes problems when running pytest - the latter tries to @@ -181,77 +177,10 @@ cd ~- CERTBOT_DOCS=1 certbot --help all > certbot/docs/cli-help.txt jws --help > acme/docs/jws-help.txt -cd .. -# freeze before installing anything else, so that we know end-user KGS -# make sure "twine upload" doesn't catch "kgs" -if [ -d kgs ] ; then - echo Deleting old kgs... - rm -rf kgs -fi -mkdir kgs -kgs="kgs/$version" -pip freeze | tee $kgs -python ../tools/pip_install.py pytest -cd ~- -for module in $SUBPKGS ; do - echo testing $module - # use an empty configuration file rather than the one in the repo root - pytest -c <(echo '') $module -done - -# pin pip hashes of the things we just built -for pkg in $SUBPKGS_IN_AUTO ; do - echo $pkg==$version \\ - pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python -c 'from sys import stdin; input = stdin.read(); print(" ", input.replace("\n--hash", " \\\n --hash"), end="")' -done > letsencrypt-auto-source/pieces/certbot-requirements.txt deactivate -# there should be one requirement specifier and two hashes for each subpackage -expected_count=$(expr $(echo $SUBPKGS_IN_AUTO | wc -w) \* 3) -if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*$expected_count " ; then - echo Unexpected pip hash output - exit 1 -fi -# ensure we have the latest built version of leauto -letsencrypt-auto-source/build.py - -# Now we have to sign the built version of leauto. -SignLEAuto() { - yubico-piv-tool -a verify-pin --sign -s 9c -i letsencrypt-auto-source/letsencrypt-auto -o letsencrypt-auto-source/letsencrypt-auto.sig -} - -# Loop until letsencrypt-auto is signed correctly. -SignLEAuto || true -while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ - letsencrypt-auto-source/letsencrypt-auto.sig \ - letsencrypt-auto-source/letsencrypt-auto ; do - echo "The signature on letsencrypt-auto is not correct." - read -p "Would you like this script to try and sign it again [Y/n]?" response - case $response in - [yY][eE][sS]|[yY]|"") - SignLEAuto || true;; - *) - ;; - esac -done - -# This signature is not quite as strong, but easier for people to verify out of band -while ! gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto; do - echo "Unable to sign letsencrypt-auto using $RELEASE_KEY." - echo "Make sure your OpenPGP card is in your computer if you are using one." - echo "You may need to take the card out and put it back in again." - read -p "Press enter to try signing again." -done -# We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, -# but we can use the right name for certbot-auto.asc from day one -mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc - -# copy leauto to the root, overwriting the previous release version -cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto -cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto - -git add certbot-auto letsencrypt-auto letsencrypt-auto-source certbot/docs/cli-help.txt +git add certbot/docs/cli-help.txt while ! git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"; do echo "Unable to sign the release commit using git." echo "You may have to configure git to use gpg2 by running:" @@ -260,15 +189,6 @@ while ! git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"; do done git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" -cd .. -echo Now in $PWD -name=${root_without_le%.*} -ext="${root_without_le##*.}" -rev="$(git rev-parse --short HEAD)" -echo tar cJvf $name.$rev.tar.xz $name.$rev -echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz -cd ~- - # Add master section to CHANGELOG.md header=$(head -n 4 certbot/CHANGELOG.md) body=$(sed s/nextversion/$nextversion/ tools/_changelog_top.txt) @@ -281,17 +201,7 @@ $footer" > certbot/CHANGELOG.md git add certbot/CHANGELOG.md git commit -m "Add contents to certbot/CHANGELOG.md for next version" -echo "New root: $root" -echo "Test commands (in the letstest repo):" -echo 'python multitester.py auto_targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' -echo 'python multitester.py auto_targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' -echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' -echo "In order to upload packages run the following command:" -echo twine upload "$root/dist.$version/*/*" - if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then SetVersion "$nextversion".dev0 - letsencrypt-auto-source/build.py - git add letsencrypt-auto-source/letsencrypt-auto git commit -m "Bump version to $nextversion" fi diff --git a/tools/certbot_constraints.txt b/tools/certbot_constraints.txt deleted file mode 100644 index 77bfef9db..000000000 --- a/tools/certbot_constraints.txt +++ /dev/null @@ -1,262 +0,0 @@ -# This is the flattened list of pinned packages to build certbot deployable artifacts. -# To generate this, do (with docker and package hashin installed): -# ``` -# tools/rebuild_certbot_contraints.py \ -# tools/certbot_constraints.txt -# ``` -# If you want to update a single dependency, run commands similar to these: -# ``` -# pip install hashin -# hashin -r dependency-requirements.txt cryptography==1.5.2 -# ``` -ConfigArgParse==1.2.3 \ - --hash=sha256:edd17be986d5c1ba2e307150b8e5f5107aba125f3574dddd02c85d5cdcfd37dc -certifi==2020.12.5 \ - --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ - --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 -cffi==1.14.4 \ - --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \ - --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \ - --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \ - --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \ - --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \ - --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \ - --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \ - --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \ - --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \ - --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \ - --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \ - --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \ - --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \ - --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \ - --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \ - --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \ - --hash=sha256:7ef7d4ced6b325e92eb4d3502946c78c5367bc416398d387b39591532536734e \ - --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \ - --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \ - --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \ - --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \ - --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \ - --hash=sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01 \ - --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \ - --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \ - --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \ - --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \ - --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \ - --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \ - --hash=sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e \ - --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \ - --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \ - --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \ - --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \ - --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \ - --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \ - --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f -chardet==4.0.0 \ - --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ - --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 -configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 -cryptography==3.3.2 \ - --hash=sha256:0d7b69674b738068fa6ffade5c962ecd14969690585aaca0a1b1fc9058938a72 \ - --hash=sha256:1bd0ccb0a1ed775cd7e2144fe46df9dc03eefd722bbcf587b3e0616ea4a81eff \ - --hash=sha256:3c284fc1e504e88e51c428db9c9274f2da9f73fdf5d7e13a36b8ecb039af6e6c \ - --hash=sha256:49570438e60f19243e7e0d504527dd5fe9b4b967b5a1ff21cc12b57602dd85d3 \ - --hash=sha256:541dd758ad49b45920dda3b5b48c968f8b2533d8981bcdb43002798d8f7a89ed \ - --hash=sha256:5a60d3780149e13b7a6ff7ad6526b38846354d11a15e21068e57073e29e19bed \ - --hash=sha256:7951a966613c4211b6612b0352f5bf29989955ee592c4a885d8c7d0f830d0433 \ - --hash=sha256:922f9602d67c15ade470c11d616f2b2364950602e370c76f0c94c94ae672742e \ - --hash=sha256:a0f0b96c572fc9f25c3f4ddbf4688b9b38c69836713fb255f4a2715d93cbaf44 \ - --hash=sha256:a777c096a49d80f9d2979695b835b0f9c9edab73b59e4ceb51f19724dda887ed \ - --hash=sha256:a9a4ac9648d39ce71c2f63fe7dc6db144b9fa567ddfc48b9fde1b54483d26042 \ - --hash=sha256:aa4969f24d536ae2268c902b2c3d62ab464b5a66bcb247630d208a79a8098e9b \ - --hash=sha256:c7390f9b2119b2b43160abb34f63277a638504ef8df99f11cb52c1fda66a2e6f \ - --hash=sha256:e18e6ab84dfb0ab997faf8cca25a86ff15dfea4027b986322026cc99e0a892da -distro==1.5.0 \ - --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ - --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 -idna==2.10 \ - --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ - --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 -josepy==1.6.0 \ - --hash=sha256:0aab1c3ceffe045e7fd5bcfe7685e27e9d2758518d9ba7116b5de34087e70bf5 \ - --hash=sha256:65f077fc5902aca1e140ddb000e7abb081d5fb8421db60b6071076ef81c5bd27 -parsedatetime==2.6 \ - --hash=sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455 \ - --hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b -pyOpenSSL==20.0.1 \ - --hash=sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51 \ - --hash=sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b -pyRFC3339==1.1 \ - --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ - --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a -pycparser==2.20 \ - --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ - --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 -pyparsing==2.4.7 \ - --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ - --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b -python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 -pytz==2021.1 \ - --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \ - --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 -requests==2.25.1 \ - --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ - --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e -requests-toolbelt==0.9.1 \ - --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ - --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 -six==1.15.0 \ - --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ - --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced -urllib3==1.26.3 \ - --hash=sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80 \ - --hash=sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73 -zope.component==4.6.2 \ - --hash=sha256:607628e4c84f7887a69a958542b5c304663e726b73aba0882e3a3f059bff14f3 \ - --hash=sha256:91628918218b3e6f6323de2a7b845e09ddc5cae131c034896c051b084bba3c92 -zope.deferredimport==4.3.1 \ - --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \ - --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a -zope.deprecation==4.4.0 \ - --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ - --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 -zope.event==4.5.0 \ - --hash=sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42 \ - --hash=sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330 -zope.hookable==5.0.1 \ - --hash=sha256:0194b9b9e7f614abba60c90b231908861036578297515d3d6508eb10190f266d \ - --hash=sha256:0c2977473918bdefc6fa8dfb311f154e7f13c6133957fe649704deca79b92093 \ - --hash=sha256:17b8bdb3b77e03a152ca0d5ca185a7ae0156f5e5a2dbddf538676633a1f7380f \ - --hash=sha256:29d07681a78042cdd15b268ae9decffed9ace68a53eebeb61d65ae931d158841 \ - --hash=sha256:36fb1b35d1150267cb0543a1ddd950c0bc2c75ed0e6e92e3aaa6ac2e29416cb7 \ - --hash=sha256:3aed60c2bb5e812bbf9295c70f25b17ac37c233f30447a96c67913ba5073642f \ - --hash=sha256:3cac1565cc768911e72ca9ec4ddf5c5109e1fef0104f19f06649cf1874943b60 \ - --hash=sha256:3d4bc0cc4a37c3cd3081063142eeb2125511db3c13f6dc932d899c512690378e \ - --hash=sha256:3f73096f27b8c28be53ffb6604f7b570fbbb82f273c6febe5f58119009b59898 \ - --hash=sha256:522d1153d93f2d48aa0bd9fb778d8d4500be2e4dcf86c3150768f0e3adbbc4ef \ - --hash=sha256:523d2928fb7377bbdbc9af9c0b14ad73e6eaf226349f105733bdae27efd15b5a \ - --hash=sha256:5848309d4fc5c02150a45e8f8d2227e5bfda386a508bbd3160fed7c633c5a2fa \ - --hash=sha256:6781f86e6d54a110980a76e761eb54590630fd2af2a17d7edf02a079d2646c1d \ - --hash=sha256:6fd27921ebf3aaa945fa25d790f1f2046204f24dba4946f82f5f0a442577c3e9 \ - --hash=sha256:70d581862863f6bf9e175e85c9d70c2d7155f53fb04dcdb2f73cf288ca559a53 \ - --hash=sha256:81867c23b0dc66c8366f351d00923f2bc5902820a24c2534dfd7bf01a5879963 \ - --hash=sha256:81db29edadcbb740cd2716c95a297893a546ed89db1bfe9110168732d7f0afdd \ - --hash=sha256:86bd12624068cea60860a0759af5e2c3adc89c12aef6f71cf12f577e28deefe3 \ - --hash=sha256:9c184d8f9f7a76e1ced99855ccf390ffdd0ec3765e5cbf7b9cada600accc0a1e \ - --hash=sha256:acc789e8c29c13555e43fe4bf9fcd15a65512c9645e97bbaa5602e3201252b02 \ - --hash=sha256:afaa740206b7660d4cc3b8f120426c85761f51379af7a5b05451f624ad12b0af \ - --hash=sha256:b5f5fa323f878bb16eae68ea1ba7f6c0419d4695d0248bed4b18f51d7ce5ab85 \ - --hash=sha256:bd89e0e2c67bf4ac3aca2a19702b1a37269fb1923827f68324ac2e7afd6e3406 \ - --hash=sha256:c212de743283ec0735db24ec6ad913758df3af1b7217550ff270038062afd6ae \ - --hash=sha256:ca553f524293a0bdea05e7f44c3e685e4b7b022cb37d87bc4a3efa0f86587a8d \ - --hash=sha256:cab67065a3db92f636128d3157cc5424a145f82d96fb47159c539132833a6d36 \ - --hash=sha256:d3b3b3eedfdbf6b02898216e85aa6baf50207f4378a2a6803d6d47650cd37031 \ - --hash=sha256:d9f4a5a72f40256b686d31c5c0b1fde503172307beb12c1568296e76118e402c \ - --hash=sha256:df5067d87aaa111ed5d050e1ee853ba284969497f91806efd42425f5348f1c06 \ - --hash=sha256:e2587644812c6138f05b8a41594a8337c6790e3baf9a01915e52438c13fc6bef \ - --hash=sha256:e27fd877662db94f897f3fd532ef211ca4901eb1a70ba456f15c0866a985464a \ - --hash=sha256:e427ebbdd223c72e06ba94c004bb04e996c84dec8a0fa84e837556ae145c439e \ - --hash=sha256:e583ad4309c203ef75a09d43434cf9c2b4fa247997ecb0dcad769982c39411c7 \ - --hash=sha256:e760b2bc8ece9200804f0c2b64d10147ecaf18455a2a90827fbec4c9d84f3ad5 \ - --hash=sha256:ea9a9cc8bcc70e18023f30fa2f53d11ae069572a162791224e60cd65df55fb69 \ - --hash=sha256:ecb3f17dce4803c1099bd21742cd126b59817a4e76a6544d31d2cca6e30dbffd \ - --hash=sha256:ed794e3b3de42486d30444fb60b5561e724ee8a2d1b17b0c2e0f81e3ddaf7a87 \ - --hash=sha256:ee885d347279e38226d0a437b6a932f207f691c502ee565aba27a7022f1285df \ - --hash=sha256:fd5e7bc5f24f7e3d490698f7b854659a9851da2187414617cd5ed360af7efd63 \ - --hash=sha256:fe45f6870f7588ac7b2763ff1ce98cce59369717afe70cc353ec5218bc854bcc -zope.interface==5.2.0 \ - --hash=sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1 \ - --hash=sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d \ - --hash=sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123 \ - --hash=sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232 \ - --hash=sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549 \ - --hash=sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102 \ - --hash=sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5 \ - --hash=sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45 \ - --hash=sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00 \ - --hash=sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc \ - --hash=sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7 \ - --hash=sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104 \ - --hash=sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034 \ - --hash=sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3 \ - --hash=sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3 \ - --hash=sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4 \ - --hash=sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86 \ - --hash=sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96 \ - --hash=sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546 \ - --hash=sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb \ - --hash=sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3 \ - --hash=sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b \ - --hash=sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b \ - --hash=sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec \ - --hash=sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae \ - --hash=sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e \ - --hash=sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386 \ - --hash=sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2 \ - --hash=sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a \ - --hash=sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d \ - --hash=sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a \ - --hash=sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24 \ - --hash=sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d \ - --hash=sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b \ - --hash=sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50 \ - --hash=sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523 \ - --hash=sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a \ - --hash=sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095 \ - --hash=sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a \ - --hash=sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520 \ - --hash=sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65 \ - --hash=sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11 \ - --hash=sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c \ - --hash=sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7 \ - --hash=sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332 \ - --hash=sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e \ - --hash=sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c \ - --hash=sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7 \ - --hash=sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20 \ - --hash=sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc \ - --hash=sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd \ - --hash=sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537 -zope.proxy==4.3.5 \ - --hash=sha256:00573dfa755d0703ab84bb23cb6ecf97bb683c34b340d4df76651f97b0bab068 \ - --hash=sha256:092049280f2848d2ba1b57b71fe04881762a220a97b65288bcb0968bb199ec30 \ - --hash=sha256:0cbd27b4d3718b5ec74fc65ffa53c78d34c65c6fd9411b8352d2a4f855220cf1 \ - --hash=sha256:17fc7e16d0c81f833a138818a30f366696653d521febc8e892858041c4d88785 \ - --hash=sha256:19577dfeb70e8a67249ba92c8ad20589a1a2d86a8d693647fa8385408a4c17b0 \ - --hash=sha256:207aa914576b1181597a1516e1b90599dc690c095343ae281b0772e44945e6a4 \ - --hash=sha256:219a7db5ed53e523eb4a4769f13105118b6d5b04ed169a283c9775af221e231f \ - --hash=sha256:2b50ea79849e46b5f4f2b0247a3687505d32d161eeb16a75f6f7e6cd81936e43 \ - --hash=sha256:5903d38362b6c716e66bbe470f190579c530a5baf03dbc8500e5c2357aa569a5 \ - --hash=sha256:5c24903675e271bd688c6e9e7df5775ac6b168feb87dbe0e4bcc90805f21b28f \ - --hash=sha256:5ef6bc5ed98139e084f4e91100f2b098a0cd3493d4e76f9d6b3f7b95d7ad0f06 \ - --hash=sha256:61b55ae3c23a126a788b33ffb18f37d6668e79a05e756588d9e4d4be7246ab1c \ - --hash=sha256:63ddb992931a5e616c87d3d89f5a58db086e617548005c7f9059fac68c03a5cc \ - --hash=sha256:6943da9c09870490dcfd50c4909c0cc19f434fa6948f61282dc9cb07bcf08160 \ - --hash=sha256:6ad40f85c1207803d581d5d75e9ea25327cd524925699a83dfc03bf8e4ba72b7 \ - --hash=sha256:6b44433a79bdd7af0e3337bd7bbcf53dd1f9b0fa66bf21bcb756060ce32a96c1 \ - --hash=sha256:6bbaa245015d933a4172395baad7874373f162955d73612f0b66b6c2c33b6366 \ - --hash=sha256:7007227f4ea85b40a2f5e5a244479f6a6dfcf906db9b55e812a814a8f0e2c28d \ - --hash=sha256:74884a0aec1f1609190ec8b34b5d58fb3b5353cf22b96161e13e0e835f13518f \ - --hash=sha256:7d25fe5571ddb16369054f54cdd883f23de9941476d97f2b92eb6d7d83afe22d \ - --hash=sha256:7e162bdc5e3baad26b2262240be7d2bab36991d85a6a556e48b9dfb402370261 \ - --hash=sha256:814d62678dc3a30f4aa081982d830b7c342cf230ffc9d030b020cb154eeebf9e \ - --hash=sha256:8878a34c5313ee52e20aa50b03138af8d472bae465710fb954d133a9bfd3c38d \ - --hash=sha256:a66a0d94e5b081d5d695e66d6667e91e74d79e273eee95c1747717ba9cb70792 \ - --hash=sha256:a69f5cbf4addcfdf03dda564a671040127a6b7c34cf9fe4973582e68441b63fa \ - --hash=sha256:b00f9f0c334d07709d3f73a7cb8ae63c6ca1a90c790a63b5e7effa666ef96021 \ - --hash=sha256:b6ed71e4a7b4690447b626f499d978aa13197a0e592950e5d7020308f6054698 \ - --hash=sha256:bdf5041e5851526e885af579d2f455348dba68d74f14a32781933569a327fddf \ - --hash=sha256:be034360dd34e62608419f86e799c97d389c10a0e677a25f236a971b2f40dac9 \ - --hash=sha256:cc8f590a5eed30b314ae6b0232d925519ade433f663de79cc3783e4b10d662ba \ - --hash=sha256:cd7a318a15fe6cc4584bf3c4426f092ed08c0fd012cf2a9173114234fe193e11 \ - --hash=sha256:cf19b5f63a59c20306e034e691402b02055c8f4e38bf6792c23cad489162a642 \ - --hash=sha256:cfc781ce442ec407c841e9aa51d0e1024f72b6ec34caa8fdb6ef9576d549acf2 \ - --hash=sha256:dea9f6f8633571e18bc20cad83603072e697103a567f4b0738d52dd0211b4527 \ - --hash=sha256:e4a86a1d5eb2cce83c5972b3930c7c1eac81ab3508464345e2b8e54f119d5505 \ - --hash=sha256:e7106374d4a74ed9ff00c46cc00f0a9f06a0775f8868e423f85d4464d2333679 \ - --hash=sha256:e98a8a585b5668aa9e34d10f7785abf9545fe72663b4bfc16c99a115185ae6a5 \ - --hash=sha256:f64840e68483316eb58d82c376ad3585ca995e69e33b230436de0cdddf7363f9 \ - --hash=sha256:f8f4b0a9e6683e43889852130595c8854d8ae237f2324a053cdd884de936aa9b \ - --hash=sha256:fc45a53219ed30a7f670a6d8c98527af0020e6fd4ee4c0a8fb59f147f06d816c diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 4f5eda34e..f6bdea58c 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,7 +1,7 @@ # Specifies Python package versions for development and building Docker images. -# It includes in particular packages not specified in letsencrypt-auto's requirements file. +# It includes in particular packages not specified in tools/certbot_constraints.txt. # Some dev package versions specified here may be overridden by higher level constraints -# files during tests (eg. tools/certbot_constraints.txt). +# files during tests (eg. tools/oldest_constraints.txt). alabaster==0.7.10 apacheconfig==0.3.2 apipkg==1.4 @@ -60,7 +60,7 @@ MarkupSafe==1.1.1 mccabe==0.6.1 more-itertools==5.0.0 msrest==0.6.18 -mypy==0.710 +mypy==0.812 mypy-extensions==0.4.3 ndg-httpsclient==0.3.2 oauth2client==4.0.0 @@ -83,10 +83,10 @@ PyGithub==1.52 Pygments==2.2.0 pyjwt==1.7.1 pylint==2.4.3 +pynacl==1.3.0 # If pynsist version is upgraded, our NSIS template windows-installer/template.nsi # must be upgraded if necessary using the new built-in one from pynsist. -pynacl==1.3.0 -pynsist==2.6 +pynsist==2.7 pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 @@ -120,6 +120,7 @@ traitlets==4.3.3 twine==1.11.0 typed-ast==1.4.1 typing==3.6.4 +typing-extensions==3.7.4.3 uritemplate==3.0.0 virtualenv==16.6.2 wcwidth==0.1.8 diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index d2ebe3537..14bd3323a 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -27,18 +27,17 @@ RUN apk add --no-cache --virtual .certbot-deps \ binutils # Install certbot from sources -# -# We don't use tools/pip_install.py below so the hashes in -# dependency-requirements.txt can be used when installing packages for extra -# security. RUN apk add --no-cache --virtual .build-deps \ gcc \ linux-headers \ openssl-dev \ musl-dev \ libffi-dev \ + python3-dev \ + cargo \ && python tools/pipstrap.py \ && python tools/pip_install.py --no-cache-dir \ --editable src/acme \ --editable src/certbot \ - && apk del .build-deps + && apk del .build-deps \ + && rm -rf ${HOME}/.cargo diff --git a/tools/eff-pubkey.pem b/tools/eff-pubkey.pem deleted file mode 100644 index fe6c2f5bb..000000000 --- a/tools/eff-pubkey.pem +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq -OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 -xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp -9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij -n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH -cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ -CQIDAQAB ------END PUBLIC KEY----- diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index f6528f396..c0f114639 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -59,9 +59,6 @@ zope.hookable==4.0.4 # Ubuntu Bionic constraints. cryptography==2.1.4 distro==1.0.1 -# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins -# using their local-oldest-requirements.txt -dns-lexicon==2.2.1 httplib2==0.9.2 idna==2.6 setuptools==39.0.1 @@ -77,3 +74,6 @@ parsedatetime==2.4 # Tracking at https://github.com/certbot/certbot/issues/6473 boto3==1.4.7 botocore==1.7.41 +# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins +# using their local-oldest-requirements.txt +dns-lexicon==3.1.0 diff --git a/tools/pinning/pin.sh b/tools/pinning/pin.sh new file mode 100755 index 000000000..ce47f02a0 --- /dev/null +++ b/tools/pinning/pin.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# This script accepts no arguments and automates the process of updating +# Certbot's dependencies. Dependencies can be pinned to older versions by +# modifying pyproject.toml in the same directory as this file. +set -euo pipefail + +WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +REPO_ROOT="$(dirname "$(dirname "${WORK_DIR}")")" +PIPSTRAP_CONSTRAINTS="${REPO_ROOT}/tools/pipstrap_constraints.txt" +RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")" +REQUIREMENTS_FILE="$REPO_ROOT/tools/requirements.txt" +STRIP_HASHES="${REPO_ROOT}/tools/strip_hashes.py" + +if ! command -v poetry >/dev/null; then + echo "Please install poetry." + echo "You may need to recreate Certbot's virtual environment and activate it." + exit 1 +fi + +cd "${WORK_DIR}" + +if [ -f poetry.lock ]; then + rm poetry.lock +fi + +poetry lock + +TEMP_REQUIREMENTS=$(mktemp) +trap 'rm poetry.lock; rm $TEMP_REQUIREMENTS' EXIT + +poetry export -o "${TEMP_REQUIREMENTS}" --without-hashes +# We need to remove local packages from the requirements file. +sed -i '/^acme @/d; /certbot/d;' "${TEMP_REQUIREMENTS}" +# Poetry currently will not include pip, setuptools, or wheel in lockfiles or +# requirements files. This was resolved by +# https://github.com/python-poetry/poetry/pull/2826, but as of writing this it +# hasn't been included in a release yet. For now, we continue to keep +# pipstrap's pinning separate which has the added benefit of having it continue +# to check hashes when pipstrap is run directly. +"${STRIP_HASHES}" "${PIPSTRAP_CONSTRAINTS}" >> "${TEMP_REQUIREMENTS}" + +cat << EOF > "$REQUIREMENTS_FILE" +# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using +# that script. +# +# It is normally used as constraints to pip, however, it has the name +# requirements.txt so that is scanned by GitHub. See +# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems +# for more info. +EOF +cat "${TEMP_REQUIREMENTS}" >> "${REQUIREMENTS_FILE}" diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml new file mode 100644 index 000000000..4dee88d51 --- /dev/null +++ b/tools/pinning/pyproject.toml @@ -0,0 +1,90 @@ +[tool.poetry] +name = "certbot-pinner" +version = "0.1.0" +description = "A simple project for pinning Certbot's dependencies using Poetry." +authors = ["Certbot Project"] +license = "Apache License 2.0" + +[tool.poetry.dependencies] +python = "^3.6" + +# Local dependencies +# Any local packages that have dependencies on other local packages must be +# listed below before the package it depends on. For instance, certbot depends +# on acme so certbot must be listed before acme. +certbot-ci = {path = "../../certbot-ci", extras = ["docs"]} +certbot-compatibility-test = {path = "../../certbot-compatibility-test", extras = ["docs"]} +certbot-dns-cloudflare = {path = "../../certbot-dns-cloudflare", extras = ["docs"]} +certbot-dns-cloudxns = {path = "../../certbot-dns-cloudxns", extras = ["docs"]} +certbot-dns-digitalocean = {path = "../../certbot-dns-digitalocean", extras = ["docs"]} +certbot-dns-dnsimple = {path = "../../certbot-dns-dnsimple", extras = ["docs"]} +certbot-dns-dnsmadeeasy = {path = "../../certbot-dns-dnsmadeeasy", extras = ["docs"]} +certbot-dns-gehirn = {path = "../../certbot-dns-gehirn", extras = ["docs"]} +certbot-dns-google = {path = "../../certbot-dns-google", extras = ["docs"]} +certbot-dns-linode = {path = "../../certbot-dns-linode", extras = ["docs"]} +certbot-dns-luadns = {path = "../../certbot-dns-luadns", extras = ["docs"]} +certbot-dns-nsone = {path = "../../certbot-dns-nsone", extras = ["docs"]} +certbot-dns-ovh = {path = "../../certbot-dns-ovh", extras = ["docs"]} +certbot-dns-rfc2136 = {path = "../../certbot-dns-rfc2136", extras = ["docs"]} +certbot-dns-route53 = {path = "../../certbot-dns-route53", extras = ["docs"]} +certbot-dns-sakuracloud = {path = "../../certbot-dns-sakuracloud", extras = ["docs"]} +certbot-nginx = {path = "../../certbot-nginx", extras = ["docs"]} +certbot-apache = {path = "../../certbot-apache", extras = ["dev"]} +certbot = {path = "../../certbot", extras = ["dev", "docs"]} +acme = {path = "../../acme", extras = ["dev", "docs"]} +letstest = {path = "../../letstest"} +windows-installer = {path = "../../windows-installer"} + +# Extra dependencies +# awscli is just listed here as a performance optimization. As of writing this, +# there are some conflicts in shared dependencies between it and other packages +# we depend on. To try and resolve them, poetry searches through older versions +# of awscli to see if that resolves the conflict, but there are over 1000 +# versions of awscli on PyPI so this process takes too long. Providing a high +# minimum version here prevents poetry from searching this path and it fairly +# quickly resolves the dependency conflict in another way. +awscli = ">=1.19.62" +# As of writing this, cython is a build dependency of pyyaml. Since there +# doesn't appear to be a good way to automatically track down and pin build +# dependencies in Python (see +# https://discuss.python.org/t/how-to-pin-build-dependencies/8238), we list it +# as a dependency here to ensure a version of cython is pinned for extra +# stability. +cython = "*" +# We install mock in our "external-mock" tox environment to test that we didn't +# break Certbot's test API which used to always use mock objects from the 3rd +# party mock library. We list the mock dependency here so that is pinned, but +# we don't depend on it in Certbot to avoid installing mock when it's not +# needed. This dependency can be removed here once Certbot's support for the +# 3rd party mock library has been dropped. +mock = "*" +# Upgrading coverage, pylint, pytest, and some of pytest's plugins causes many +# test failures so let's pin these packages back for now. +coverage = "4.5.4" +pylint = "2.4.3" +pytest = "3.2.5" +pytest-forked = "0.2" +# We were originally pinning back python-augeas for certbot-auto because we +# found the way older versions of the library linked to Augeas were more +# reliable. That's no longer a concern, however, we continue to pin back the +# library for now because it causes Certbot tests on Windows to fail. See +# https://github.com/certbot/certbot/issues/8732. +python-augeas = "0.5.0" +# Because some parts of Certbot documentation are generated from zope.interfaces, +# we need the Sphinx plugin repoze.sphinx.autointerface. This plugin is not +# currently compatible with Sphinx>=4 though so we're pinning an older version +# for now. See https://github.com/repoze/repoze.sphinx.autointerface/issues/16 +# for more info. This pinning could also be removed by the removal of +# zope.interface in Certbot. +sphinx = "<4" +# Library traitlets is a transitive dependency of ipdb (traitlets -> ipython -> ipdb). +# Version 5.x is incompatible with Python 3.6 but for some reasons, poetry fails to +# add the appropriate marker and allows this version to be installed under Python 3.6. +# We add a pinning to not create a set of requirements incompatible with Python 3.6. +traitlets = "<5" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/tools/pip_install.py b/tools/pip_install.py index e06650ff2..054ca1229 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -3,7 +3,7 @@ # to 1, a combination of tools/oldest_constraints.txt, # tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the # top level of the package's directory is used, otherwise, a combination of -# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# tools/certbot_constraints.txt and tools/dev_constraints.txt is used. The # other file always takes precedence over tools/dev_constraints.txt. If # CERTBOT_OLDEST is set, this script must be run with `-e ` and # no other arguments. @@ -39,51 +39,34 @@ def find_tools_path(): return os.path.dirname(readlink.main(__file__)) -def certbot_oldest_processing(tools_path, args, test_constraints): +def certbot_oldest_processing(tools_path, args, constraints_path): if args[0] != '-e' or len(args) != 2: raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' 'with a single -e argument.') # remove any extras such as [dev] pkg_dir = re.sub(r'\[\w+\]', '', args[1]) + # The order of the files in this list matters as files specified later can + # override the pinnings found in earlier files. + pinning_files = [os.path.join(tools_path, 'dev_constraints.txt'), + os.path.join(tools_path, 'oldest_constraints.txt')] requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') - shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) # packages like acme don't have any local oldest requirements - if not os.path.isfile(requirements): - return None - + if os.path.isfile(requirements): + # We add requirements to the end of the list so it can override + # anything that it needs to. + pinning_files.append(requirements) + else: + requirements = None + with open(constraints_path, 'w') as fd: + fd.write(merge_module.main(*pinning_files)) return requirements -def certbot_normal_processing(tools_path, test_constraints): +def certbot_normal_processing(tools_path, constraints_path): repo_path = os.path.dirname(tools_path) - certbot_requirements = os.path.normpath(os.path.join( - repo_path, 'tools/certbot_constraints.txt')) - with open(certbot_requirements, 'r') as fd: - certbot_reqs = fd.readlines() - with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd: - pipstrap_reqs = fd.readlines() - with open(test_constraints, 'w') as fd: - data_certbot = "\n".join(strip_hashes.process_entries(certbot_reqs)) - data_pipstrap = "\n".join(strip_hashes.process_entries(pipstrap_reqs)) - data = "\n".join([data_certbot, data_pipstrap]) - fd.write(data) - - -def merge_requirements(tools_path, requirements, test_constraints, all_constraints): - # Order of the files in the merge function matters. - # Indeed version retained for a given package will be the last version - # found when following all requirements in the given order. - # Here is the order by increasing priority: - # 1) The general development constraints (tools/dev_constraints.txt) - # 2) The general tests constraints (oldest_requirements.txt or - # certbot_constraints.txt + pipstrap's constraints for the normal processing) - # 3) The local requirement file, typically local-oldest-requirement in oldest tests - files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] - if requirements: - files.append(requirements) - merged_requirements = merge_module.main(*files) - with open(all_constraints, 'w') as fd: - fd.write(merged_requirements) + requirements = os.path.normpath(os.path.join( + repo_path, 'tools/requirements.txt')) + shutil.copy(requirements, constraints_path) def call_with_print(command, env=None): @@ -104,24 +87,21 @@ def main(args): tools_path = find_tools_path() with temporary_directory() as working_dir: - test_constraints = os.path.join(working_dir, 'test_constraints.txt') - all_constraints = os.path.join(working_dir, 'all_constraints.txt') - if os.environ.get('CERTBOT_NO_PIN') == '1': # With unpinned dependencies, there is no constraint pip_install_with_print(' '.join(args)) else: # Otherwise, we merge requirements to build the constraints and pin dependencies + constraints_path = os.path.join(working_dir, 'constraints.txt') requirements = None if os.environ.get('CERTBOT_OLDEST') == '1': - requirements = certbot_oldest_processing(tools_path, args, test_constraints) + requirements = certbot_oldest_processing(tools_path, args, constraints_path) else: - certbot_normal_processing(tools_path, test_constraints) + certbot_normal_processing(tools_path, constraints_path) env = os.environ.copy() - env["PIP_CONSTRAINT"] = all_constraints + env["PIP_CONSTRAINT"] = constraints_path - merge_requirements(tools_path, requirements, test_constraints, all_constraints) if requirements: # This branch is executed during the oldest tests # First step, install the transitive dependencies of oldest requirements # in respect with oldest constraints. diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index de2a0ff57..646344f3a 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# pip installs packages in editable mode using certbot-auto's requirements file +# pip installs packages in editable mode using tools/certbot_constraints.txt # as constraints # # cryptography is currently using this script in their CI at diff --git a/tools/rebuild_certbot_constraints.py b/tools/rebuild_certbot_constraints.py deleted file mode 100755 index f5e5d3ca7..000000000 --- a/tools/rebuild_certbot_constraints.py +++ /dev/null @@ -1,280 +0,0 @@ -#!/usr/bin/env python -""" -Gather and consolidate the up-to-date dependencies available and required to install certbot -on various Linux distributions. It generates a requirements file contained the pinned and hashed -versions, ready to be used by pip to install the certbot dependencies. - -This script is typically used to update the certbot_constraints.txt file. - -To achieve its purpose, this script will start a certbot installation with unpinned dependencies, -then gather them, on various distributions started as Docker containers. - -Usage: tools/rebuild_certbot_constraints.py new_requirements.txt - -NB1: Docker must be installed on the machine running this script. -NB2: Python library 'hashin' must be installed on the machine running this script. -""" -from __future__ import print_function -import re -import shutil -import subprocess -import tempfile -import os -from os.path import dirname, abspath, join -import sys -import argparse - -# The list of docker distributions to test dependencies against with. -DISTRIBUTION_LIST = [ - 'ubuntu:20.04', 'ubuntu:18.04', 'debian:buster', - 'centos:8', 'centos:7', 'fedora:29', -] - -# These constraints will be added while gathering dependencies on each distribution. -# It can be used because a particular version for a package is required for any reason, -# or to solve a version conflict between two distributions requirements. -AUTHORITATIVE_CONSTRAINTS = { - # Too touchy to move to a new version. And will be removed soon - # in favor of pure python parser for Apache. - 'python-augeas': '0.5.0', - # We avoid cryptography 3.4+ since it requires Rust to compile the wheels, and - # this needs some work on the snap builds. - 'cryptography': '3.3.2', -} - -# ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path) -CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__))) - -# The script will be used to gather dependencies for a given distribution. -# - bootstrap_os_packages.sh is used to install relevant OS packages, and set up an initial venv -# - then this venv is used to consistently construct an empty new venv -# - once pipstrap.py, this new venv pip-installs certbot runtime (including apache/nginx), -# without pinned dependencies, and respecting input authoritative requirements -# - `certbot plugins` is called to check we have a healthy environment -# - finally current set of dependencies is extracted out of the docker using pip freeze -SCRIPT = r"""#!/bin/sh -set -ex - -cd /tmp/certbot -tests/letstest/scripts/bootstrap_os_packages.sh - -python3 -m venv /tmp/venv - -/tmp/venv/bin/python tools/pipstrap.py -/tmp/venv/bin/pip install -e acme -e certbot -e certbot-apache -e certbot-nginx -c /tmp/constraints.txt -/tmp/venv/bin/certbot plugins -/tmp/venv/bin/pip freeze >> /tmp/workspace/requirements.txt -""" - - -def _read_from(file): - """Read all content of the file, and return it as a string.""" - with open(file, 'r') as file_h: - return file_h.read() - - -def _write_to(file, content): - """Write given string content to the file, overwriting its initial content.""" - with open(file, 'w') as file_h: - file_h.write(content) - - -def _requirements_from_one_distribution(distribution, verbose): - """ - Calculate the Certbot dependencies expressed for the given distribution, using the official - Docker for this distribution, and return the lines of the generated requirements file. - """ - print('===> Gathering dependencies for {0}.'.format(distribution)) - workspace = tempfile.mkdtemp() - script = join(workspace, 'script.sh') - authoritative_constraints = join(workspace, 'constraints.txt') - cid_file = join(workspace, 'cid') - - try: - _write_to(script, SCRIPT) - os.chmod(script, 0o755) - - _write_to(authoritative_constraints, '\n'.join( - '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) - - command = ['docker', 'run', '--rm', '--cidfile', cid_file, - '--network=host', - '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), - '-v', '{0}:/tmp/workspace'.format(workspace), - '-v', '{0}:/tmp/constraints.txt'.format(authoritative_constraints), - distribution, '/tmp/workspace/script.sh'] - sub_stdout = sys.stdout if verbose else subprocess.PIPE - sub_stderr = sys.stderr if verbose else subprocess.STDOUT - process = subprocess.Popen(command, stdout=sub_stdout, stderr=sub_stderr, universal_newlines=True) - stdoutdata, _ = process.communicate() - - if process.returncode: - if stdoutdata: - sys.stderr.write('Output was:\n{0}'.format(stdoutdata)) - raise RuntimeError('Error while gathering dependencies for {0}.'.format(distribution)) - - with open(join(workspace, 'requirements.txt'), 'r') as file_h: - return file_h.readlines() - finally: - if os.path.isfile(cid_file): - cid = _read_from(cid_file) - try: - subprocess.check_output(['docker', 'kill', cid], stderr=subprocess.PIPE) - except subprocess.CalledProcessError: - pass - shutil.rmtree(workspace) - - -def _parse_and_merge_requirements(dependencies_map, requirements_file_lines, distribution): - """ - Extract every requirement from the given requirements file, and merge it in the dependency map. - Merging here means that the map contain every encountered dependency, and the version used in - each distribution. - - Example: - # dependencies_map = { - # } - _parse_and_merge_requirements(['cryptography=='1.2','requests=='2.1.0'], dependencies_map, 'debian:stretch') - # dependencies_map = { - # 'cryptography': [('1.2', 'debian:stretch)], - # 'requests': [('2.1.0', 'debian:stretch')] - # } - _parse_and_merge_requirements(['requests=='2.4.0', 'mock==1.3'], dependencies_map, 'centos:7') - # dependencies_map = { - # 'cryptography': [('1.2', 'debian:stretch)], - # 'requests': [('2.1.0', 'debian:stretch'), ('2.4.0', 'centos:7')], - # 'mock': [('2.4.0', 'centos:7')] - # } - """ - for line in requirements_file_lines: - match = re.match(r'([^=]+)==([^=]+)', line.strip()) - if not line.startswith('-e') and not line.startswith('#') and match: - package, version = match.groups() - if package not in ['acme', 'certbot', 'certbot-apache', 'certbot-nginx', 'pkg-resources']: - dependencies_map.setdefault(package, []).append((version, distribution)) - - -def _consolidate_and_validate_dependencies(dependency_map): - """ - Given the dependency map of all requirements found in all distributions for Certbot, - construct an array containing the unit requirements for Certbot to be used by pip, - and the version conflicts, if any, between several distributions for a package. - Return requirements and conflicts as a tuple. - """ - print('===> Consolidate and validate the dependency map.') - requirements = [] - conflicts = [] - for package, versions in dependency_map.items(): - reduced_versions = _reduce_versions(versions) - - if len(reduced_versions) > 1: - version_list = ['{0} ({1})'.format(version, ','.join(distributions)) - for version, distributions in reduced_versions.items()] - conflict = ('package {0} is declared with several versions: {1}' - .format(package, ', '.join(version_list))) - conflicts.append(conflict) - sys.stderr.write('ERROR: {0}\n'.format(conflict)) - else: - requirements.append((package, list(reduced_versions)[0])) - - requirements.sort(key=lambda x: x[0]) - return requirements, conflicts - - -def _reduce_versions(version_dist_tuples): - """ - Get an array of version/distribution tuples, - and reduce it to a map based on the version values. - - Example: [('1.2.0', 'debian:stretch'), ('1.4.0', 'ubuntu:18.04'), ('1.2.0', 'centos:6')] - => {'1.2.0': ['debiqn:stretch', 'centos:6'], '1.4.0': ['ubuntu:18.04']} - """ - version_dist_map = {} - for version, distribution in version_dist_tuples: - version_dist_map.setdefault(version, []).append(distribution) - - return version_dist_map - - -def _write_requirements(dest_file, requirements, conflicts): - """ - Given the list of requirements and conflicts, write a well-formatted requirements file, - whose requirements are hashed signed using hashin library. Conflicts are written at the end - of the generated file. - """ - print('===> Calculating hashes for the requirement file.') - - _write_to(dest_file, '''\ -# This is the flattened list of pinned packages to build certbot deployable artifacts. -# To generate this, do (with docker and package hashin installed): -# ``` -# tools/rebuild_certbot_contraints.py \\ -# tools/certbot_constraints.txt -# ``` -# If you want to update a single dependency, run commands similar to these: -# ``` -# pip install hashin -# hashin -r dependency-requirements.txt cryptography==1.5.2 -# ``` -''') - - for req in requirements: - if req[0] in AUTHORITATIVE_CONSTRAINTS: - # If requirement is in AUTHORITATIVE_CONSTRAINTS, take its value instead of the - # computed one to get any environment descriptor that would have been added. - req = (req[0], AUTHORITATIVE_CONSTRAINTS[req[0]]) - subprocess.check_call(['hashin', '{0}=={1}'.format(req[0], req[1]), - '--requirements-file', dest_file]) - - if conflicts: - with open(dest_file, 'a') as file_h: - file_h.write('\n## ! SOME ERRORS OCCURRED ! ##\n') - file_h.write('\n'.join('# {0}'.format(conflict) for conflict in conflicts)) - file_h.write('\n') - - return _read_from(dest_file) - - -def _gather_dependencies(dest_file, verbose): - """ - Main method of this script. Given a destination file path, will write the file - containing the consolidated and hashed requirements for Certbot, validated - against several Linux distributions. - """ - dependencies_map = {} - - for distribution in DISTRIBUTION_LIST: - requirements_file_lines = _requirements_from_one_distribution(distribution, verbose) - _parse_and_merge_requirements(dependencies_map, requirements_file_lines, distribution) - - requirements, conflicts = _consolidate_and_validate_dependencies(dependencies_map) - - return _write_requirements(dest_file, requirements, conflicts) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=('Build a sanitized, pinned and hashed requirements file for certbot deployable' - ' artifacts, validated against several OS distributions using Docker.')) - parser.add_argument('requirements_path', - help='path for the generated requirements file') - parser.add_argument('--verbose', '-v', action='store_true', - help='verbose will display all output during docker execution') - - namespace = parser.parse_args() - - try: - subprocess.check_output(['hashin', '--version']) - except subprocess.CalledProcessError: - raise RuntimeError('Python library hashin is not installed in the current environment.') - - try: - subprocess.check_output(['docker', '--version'], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - raise RuntimeError('Docker is not installed or accessible to current user.') - - file_content = _gather_dependencies(namespace.requirements_path, namespace.verbose) - - print(file_content) - print('===> Rebuilt requirement file is available on path {0}' - .format(abspath(namespace.requirements_path))) diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 000000000..9e442f411 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,186 @@ +# This file was generated by tools/pinning/pin.sh and can be updated using +# that script. +# +# It is normally used as constraints to pip, however, it has the name +# requirements.txt so that is scanned by GitHub. See +# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems +# for more info. +alabaster==0.7.12; python_version >= "3.6" +apacheconfig==0.3.2; python_version >= "3.6" +apipkg==1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +appnope==0.1.2 +astroid==2.3.3; python_version >= "3.6" +attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +awscli==1.19.69; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") +azure-devops==6.0.0b4; python_version >= "3.6" +babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +backcall==0.2.0 +bcrypt==3.2.0; python_version >= "3.6" +beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" +bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +boto3==1.17.69; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.69; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +cachy==0.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +cleo==0.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +clikit==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +cloudflare==2.8.15; python_version >= "3.6" +colorama==0.4.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +configargparse==1.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +configobj==5.0.6; python_version >= "3.6" +coverage==4.5.4; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0" and python_version < "4") +crashtest==0.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +cryptography==3.4.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +cython==0.29.23; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") +decorator==5.0.7 +deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +distlib==0.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +dns-lexicon==3.6.0; python_version >= "3.6" and python_version < "4.0" +dnspython==2.1.0; python_version >= "3.6" +docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docutils==0.15.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") +execnet==1.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +fabric==2.6.0; python_version >= "3.6" +filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0" +google-api-core==1.26.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +google-api-python-client==2.3.0; python_version >= "3.6" +google-auth-httplib2==0.1.0; python_version >= "3.6" +google-auth==1.30.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +httplib2==0.19.1; python_version >= "3.6" +idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" +imagesize==1.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +importlib-resources==5.1.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" +invoke==1.5.0; python_version >= "3.6" +ipdb==0.13.7; python_version >= "3.6" +ipython-genutils==0.2.0; python_version == "3.6" +ipython==7.16.1; python_version == "3.6" +ipython==7.23.1; python_version >= "3.7" +isodate==0.6.0; python_version >= "3.6" +isort==4.3.21; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +jedi==0.18.0 +jeepney==0.6.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +jinja2==2.11.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +josepy==1.8.0; python_version >= "3.6" +jsonlines==2.0.0; python_version >= "3.6" +jsonpickle==2.0.0; python_version >= "3.6" +jsonschema==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +keyring==21.8.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +lazy-object-proxy==1.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +lockfile==0.12.2 +markupsafe==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +matplotlib-inline==0.1.2; python_version >= "3.7" +mccabe==0.6.1; python_version >= "3.6" +mock==4.0.3; python_version >= "3.6" +msgpack==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +msrest==0.6.21; python_version >= "3.6" +mypy-extensions==0.4.3; python_version >= "3.6" +mypy==0.812; python_version >= "3.6" +oauth2client==4.1.3; python_version >= "3.6" +oauthlib==3.1.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" +paramiko==2.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +parsedatetime==2.6; python_version >= "3.6" +parso==0.8.2; python_version == "3.6" +pastel==0.2.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pathlib2==2.3.5; python_version >= "3.6" +pexpect==4.8.0 +pickleshare==0.7.5 +pkginfo==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +ply==3.11; python_version >= "3.6" +poetry-core==1.0.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +poetry==1.1.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +prompt-toolkit==3.0.3 +protobuf==3.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pygithub==1.55; python_version >= "3.6" +pygments==2.9.0 +pyjwt==2.1.0; python_version >= "3.6" +pylev==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pylint==2.4.3; python_version >= "3.5" +pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +pynsist==2.7; python_version >= "3.6" +pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pypiwin32==223; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") +pyrfc3339==1.1; python_version >= "3.6" +pyrsistent==0.17.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest-cov==2.6.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest-forked==0.2 +pytest-xdist==1.24.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest==3.2.5 +python-augeas==0.5.0 +python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +python-digitalocean==1.16.0; python_version >= "3.6" +python-dotenv==0.17.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" +pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "win32" +pywin32==300; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") +pyyaml==5.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4.0" +readme-renderer==29.0; python_version >= "3.6" +repoze.sphinx.autointerface==0.8; python_version >= "3.6" +requests-download==0.1.2; python_version >= "3.6" +requests-file==1.5.1; python_version >= "3.6" and python_version < "4.0" +requests-oauthlib==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.6.0" and python_version < "4.0" +rfc3986==1.5.0; python_version >= "3.6" +rsa==4.7.2; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +s3transfer==0.4.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +six==1.16.0; python_version == "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version == "3.6" +snowballstemmer==2.1.0; python_version >= "3.6" +soupsieve==2.2.1; python_version >= "3.6" +sphinx-rtd-theme==0.5.2; python_version >= "3.6" +sphinx==3.5.4; python_version >= "3.5" +sphinxcontrib-applehelp==1.0.2; python_version >= "3.6" +sphinxcontrib-devhelp==1.0.2; python_version >= "3.6" +sphinxcontrib-htmlhelp==1.0.3; python_version >= "3.6" +sphinxcontrib-jsmath==1.0.1; python_version >= "3.6" +sphinxcontrib-qthelp==1.0.3; python_version >= "3.6" +sphinxcontrib-serializinghtml==1.1.4; python_version >= "3.6" +texttable==1.6.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +tldextract==3.1.0; python_version >= "3.6" and python_version < "4.0" +toml==0.10.2; python_version == "3.6" and python_full_version < "3.0.0" or python_version > "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.5.0" or python_version > "3.6" and python_full_version >= "3.5.0" +tomlkit==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +tox==3.23.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +tqdm==4.60.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +traitlets==4.3.3 +twine==3.3.0; python_version >= "3.6" +typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6" +typing-extensions==3.10.0.0; python_version >= "3.6" +uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +urllib3==1.26.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" +virtualenv==20.4.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +wcwidth==0.2.5; python_version == "3.6" +webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +websocket-client==0.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +wrapt==1.11.2; python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0") +yarg==0.1.9; python_version >= "3.6" +zipp==3.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" +zope.component==5.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.event==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.hookable==5.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.interface==5.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pip==20.2.4 +setuptools==54.1.2 +wheel==0.35.1 diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index 69c704e45..91854e9e0 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse import datetime +import functools import glob from multiprocessing import Manager from multiprocessing import Pool @@ -13,6 +14,7 @@ from os.path import exists from os.path import join from os.path import realpath import re +import shutil import subprocess import sys import tempfile @@ -27,34 +29,74 @@ CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] +# In Python, stdout and stderr are buffered in each process by default. When +# printing output from multiple processes, this can cause delays in printing +# output with lines from different processes being interleaved depending +# on when the output for that process is flushed. To prevent this, we override +# print so that it always flushes its output. Disabling output buffering can +# also be done through command line flags or environment variables set when the +# Python process starts, but this approach was taken instead to ensure +# consistent behavior regardless of how the script is invoked. +print = functools.partial(print, flush=True) + + +def _snap_log_name(target: str, arch: str): + return f'{target}_{arch}.txt' + + def _execute_build( target: str, archs: Set[str], status: Dict[str, Dict[str, str]], workspace: str) -> Tuple[int, List[str]]: - with tempfile.TemporaryDirectory() as tempdir: - environ = os.environ.copy() - environ['XDG_CACHE_HOME'] = tempdir - process = subprocess.Popen([ - 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', - '--build-on', ','.join(archs)], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True, env=environ, cwd=workspace) + # Snapcraft remote-build has a recover feature, that will make it reconnect to an existing + # build on Launchpad if possible. However, the signature used to retrieve a potential + # build is not based on the content of the sources used to build a snap, but on a hash + # of the snapcraft current working directory (the path itself, not the content). + # It means that every build started from /my/path/to/certbot will always be considered + # as the same build, whatever the actual sources are. + # To circumvent this, we create a temporary folder and use it as a workspace to build + # the snap: this path is random, making the recover feature effectively noop. + with tempfile.TemporaryDirectory() as temp_workspace: + ignore = None + if target == 'certbot': + ignore = shutil.ignore_patterns(".git", "venv*", ".tox") + shutil.copytree(workspace, temp_workspace, + dirs_exist_ok=True, symlinks=True, ignore=ignore) # type:ignore - process_output: List[str] = [] - for line in process.stdout: - process_output.append(line) - _extract_state(target, line, status) + with tempfile.TemporaryDirectory() as tempdir: + environ = os.environ.copy() + environ['XDG_CACHE_HOME'] = tempdir + process = subprocess.Popen([ + 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', + '--build-on', ','.join(archs)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True, env=environ, cwd=temp_workspace) - if any(state for state in status[target].values() if state == 'Chroot problem'): - # On this error the snapcraft process stales. Let's finish it. - process.kill() + process_output: List[str] = [] + for line in process.stdout: + process_output.append(line) + _extract_state(target, line, status) - return process.wait(), process_output + if any(state for state in status[target].values() if state == 'Chroot problem'): + # On this error the snapcraft process stales. Let's finish it. + process.kill() + + process_state = process.wait() + + for path in glob.glob(join(temp_workspace, '*.snap')): + shutil.copy(path, workspace) + for arch in archs: + log_name = _snap_log_name(target, arch) + log_path = join(temp_workspace, log_name) + if exists(log_path): + shutil.copy(log_path, workspace) + + return process_state, process_output def _build_snap( target: str, archs: Set[str], status: Dict[str, Dict[str, str]], - running: Dict[str, bool], lock: Lock) -> Dict[str, str]: + running: Dict[str, bool], output_lock: Lock) -> bool: status[target] = {arch: '...' for arch in archs} if target == 'certbot': @@ -62,24 +104,19 @@ def _build_snap( else: workspace = join(CERTBOT_DIR, target) + build_success = False retry = 3 while retry: exit_code, process_output = _execute_build(target, archs, status, workspace) + with output_lock: + print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with ' + f'exit code {exit_code}.') - print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with ' - f'exit code {exit_code}.') - sys.stdout.flush() - - with lock: - dump_output = exit_code != 0 failed_archs = [arch for arch in archs if status[target][arch] != 'Successfully built'] - if any(arch for arch in archs if status[target][arch] == 'Chroot problem'): - print('Some builds failed with the status "Chroot problem".') - print('This status is known to make any future build fail until either ' - 'the source code changes or the build on Launchpad is deleted.') - print('Please fix the build appropriately before trying a new one.') - # It is useless to retry in this situation. - retry = 0 + # If the command failed or any architecture wasn't built + # successfully, let's try to print all the output about the problem + # that we can. + dump_output = exit_code != 0 or failed_archs if exit_code == 0 and not failed_archs: # We expect to have all target snaps available, or something bad happened. snaps_list = glob.glob(join(workspace, '*.snap')) @@ -88,16 +125,12 @@ def _build_snap( f'(current list: {snaps_list}).') dump_output = True else: + build_success = True break - if failed_archs: - # We expect each failed build to have a log file, or something bad happened. - for arch in failed_archs: - if not exists(join(workspace, f'{target}_{arch}.txt')): - dump_output = True - print(f'Missing output on a failed build {target} for {arch}.') if dump_output: print(f'Dumping snapcraft remote-build output build for {target}:') print('\n'.join(process_output)) + _dump_failed_build_logs(target, archs, status, workspace) # Retry the remote build if it has been interrupted (non zero status code) # or if some builds have failed. @@ -105,22 +138,27 @@ def _build_snap( running[target] = False - return {target: workspace} + return build_success def _extract_state(project: str, output: str, status: Dict[str, Dict[str, str]]) -> None: + state = status[project] + + if "Sending build data to Launchpad..." in output: + for arch in state.keys(): + state[arch] = "Sending build data" + match = re.match(r'^.*arch=(\w+)\s+state=([\w ]+).*$', output) if match: arch = match.group(1) - state = status[project] state[arch] = match.group(2) - # You need to reassign the value of status[project] here (rather than doing - # something like status[project][arch] = match.group(2)) for the state change - # to propagate to other processes. See - # https://docs.python.org/3.8/library/multiprocessing.html#proxy-objects for - # more info. - status[project] = state + # You need to reassign the value of status[project] here (rather than doing + # something like status[project][arch] = match.group(2)) for the state change + # to propagate to other processes. See + # https://docs.python.org/3.8/library/multiprocessing.html#proxy-objects for + # more info. + status[project] = state def _dump_status_helper(archs: Set[str], status: Dict[str, Dict[str, str]]) -> None: @@ -132,53 +170,45 @@ def _dump_status_helper(archs: Set[str], status: Dict[str, Dict[str, str]]) -> N print(f'|{"-" * 26}' * len(headers)) print() - sys.stdout.flush() - def _dump_status( archs: Set[str], status: Dict[str, Dict[str, str]], - running: Dict[str, bool]) -> None: + running: Dict[str, bool], output_lock: Lock) -> None: while any(running.values()): - print(f'Remote build status at {datetime.datetime.now()}') - _dump_status_helper(archs, status) + with output_lock: + print(f'Remote build status at {datetime.datetime.now()}') + _dump_status_helper(archs, status) time.sleep(10) -def _dump_results( - targets: Set[str], archs: Set[str], status: Dict[str, Dict[str, str]], - workspaces: Dict[str, str]) -> bool: - failures = False - for target in targets: - for arch in archs: - result = status[target][arch] +def _dump_failed_build_logs( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + workspace: str) -> None: + for arch in archs: + result = status[target][arch] - if result != 'Successfully built': - failures = True + if result != 'Successfully built': + failures = True - build_output_path = join(workspaces[target], f'{target}_{arch}.txt') - if not exists(build_output_path): - build_output = f'No output has been dumped by snapcraft remote-build.' - else: - with open(join(workspaces[target], f'{target}_{arch}.txt')) as file_h: - build_output = file_h.read() + build_output_name = _snap_log_name(target, arch) + build_output_path = join(workspace, build_output_name) + if not exists(build_output_path): + build_output = f'No output has been dumped by snapcraft remote-build.' + else: + with open(build_output_path) as file_h: + build_output = file_h.read() - print(f'Output for failed build target={target} arch={arch}') - print('-------------------------------------------') - print(build_output) - print('-------------------------------------------') - print() + print(f'Output for failed build target={target} arch={arch}') + print('-------------------------------------------') + print(build_output) + print('-------------------------------------------') + print() - if not failures: - print('All builds succeeded.') - else: - print('Some builds failed.') - print() +def _dump_results(archs: Set[str], status: Dict[str, Dict[str, str]]) -> None: print(f'Results for remote build finished at {datetime.datetime.now()}') _dump_status_helper(archs, status) - return failures - def main(): parser = argparse.ArgumentParser() @@ -217,12 +247,14 @@ def main(): with manager, pool: status: Dict[str, Dict[str, str]] = manager.dict() running = manager.dict({target: True for target in targets}) - lock = manager.Lock() + # While multiple processes are running, this lock should be acquired + # before printing output. + output_lock = manager.Lock() - async_results = [pool.apply_async(_build_snap, (target, archs, status, running, lock)) + async_results = [pool.apply_async(_build_snap, (target, archs, status, running, output_lock)) for target in targets] - process = Process(target=_dump_status, args=(archs, status, running)) + process = Process(target=_dump_status, args=(archs, status, running, output_lock)) process.start() try: @@ -231,11 +263,16 @@ def main(): if process.is_alive(): raise ValueError(f"Timeout out reached ({args.timeout} seconds) during the build!") - workspaces = {} + build_success = True for async_result in async_results: - workspaces.update(async_result.get()) + if not async_result.get(): + build_success = False - if _dump_results(targets, archs, status, workspaces): + _dump_results(archs, status) + if build_success: + print('All builds succeeded.') + else: + print('Some builds failed.') raise ValueError("There were failures during the build!") finally: process.terminate() diff --git a/tools/snap/generate_dnsplugins_all.sh b/tools/snap/generate_dnsplugins_all.sh index 976b0dd7b..769b4ef4d 100755 --- a/tools/snap/generate_dnsplugins_all.sh +++ b/tools/snap/generate_dnsplugins_all.sh @@ -9,8 +9,5 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_snapcraft.sh $PLUGIN_PATH bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH # Create constraints file - "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_constraints.txt) \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \ - > "${PLUGIN_PATH}"/snap-constraints.txt + cp "${CERTBOT_DIR}"/tools/requirements.txt "${PLUGIN_PATH}"/snap-constraints.txt done diff --git a/tools/snap/generate_dnsplugins_snapcraft.sh b/tools/snap/generate_dnsplugins_snapcraft.sh index d93d8ec73..139c8b0d3 100755 --- a/tools/snap/generate_dnsplugins_snapcraft.sh +++ b/tools/snap/generate_dnsplugins_snapcraft.sh @@ -35,7 +35,14 @@ parts: - PIP_CONSTRAINT: \$SNAPCRAFT_PART_SRC/snap-constraints.txt - SNAP_BUILD: "True" # To build cryptography and cffi if needed - build-packages: [gcc, libffi-dev, libssl-dev, python3-dev] + build-packages: + - gcc + - git + - build-essential + - libssl-dev + - libffi-dev + - python3-dev + - cargo certbot-metadata: plugin: dump source: . diff --git a/tools/venv.py b/tools/venv.py index 9f7488008..f3f5781fa 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -44,8 +44,12 @@ REQUIREMENTS = [ '-e certbot-nginx', '-e certbot-compatibility-test', '-e certbot-ci', + '-e letstest', ] +if sys.platform == 'win32': + REQUIREMENTS.append('-e windows-installer') + VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') diff --git a/tox.ini b/tox.ini index 9f63b897c..a45766d87 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py3-cover,lint,mypy +envlist = py3-cover,lint,mypy [base] # pip installs the requested packages in editable mode @@ -151,6 +151,7 @@ commands = {[base]install_packages} python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} +// TODO: Re-enable strict checks for optionals with appropriate type corrections or code redesign. [testenv:mypy] basepython = python3 commands = @@ -181,12 +182,9 @@ commands = {[base]pip_install} acme certbot certbot-apache certbot-nginx python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata -# This is a duplication of the command line in testenv:le_auto to -# allow users to run the modification check by running `tox` [testenv:modification] commands = python {toxinidir}/tests/modification-check.py -passenv = TARGET_BRANCH [testenv:apache_compat] commands = @@ -279,36 +277,26 @@ passenv = DOCKER_* setenv = {[testenv:oldest]setenv} [testenv:test-farm-tests-base] -changedir = tests/letstest -deps = -rtests/letstest/requirements.txt +changedir = letstest +# The package to install is in the current working directory because of the +# value of changedir. +commands = {[base]pip_install} . passenv = AWS_* setenv = AWS_DEFAULT_REGION=us-east-1 [testenv:test-farm-apache2] changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} -passenv = {[testenv:test-farm-tests-base]passenv} -setenv = {[testenv:test-farm-tests-base]setenv} - -[testenv:test-farm-leauto-upgrades] -changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_leauto_upgrades.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} -passenv = {[testenv:test-farm-tests-base]passenv} -setenv = {[testenv:test-farm-tests-base]setenv} - -[testenv:test-farm-certonly-standalone] -changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_letsencrypt_auto_certonly_standalone.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} +commands = + {[testenv:test-farm-tests-base]commands} + {toxinidir}/tools/retry.sh letstest targets/apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir} passenv = {[testenv:test-farm-tests-base]passenv} setenv = {[testenv:test-farm-tests-base]setenv} [testenv:test-farm-sdists] changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} +commands = + {[testenv:test-farm-tests-base]commands} + {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir} passenv = {[testenv:test-farm-tests-base]passenv} setenv = {[testenv:test-farm-tests-base]setenv} diff --git a/windows-installer/certbot.ico b/windows-installer/assets/certbot.ico similarity index 100% rename from windows-installer/certbot.ico rename to windows-installer/assets/certbot.ico diff --git a/windows-installer/renew-down.ps1 b/windows-installer/assets/renew-down.ps1 similarity index 100% rename from windows-installer/renew-down.ps1 rename to windows-installer/assets/renew-down.ps1 diff --git a/windows-installer/renew-up.ps1 b/windows-installer/assets/renew-up.ps1 similarity index 100% rename from windows-installer/renew-up.ps1 rename to windows-installer/assets/renew-up.ps1 diff --git a/windows-installer/run.bat b/windows-installer/assets/run.bat similarity index 100% rename from windows-installer/run.bat rename to windows-installer/assets/run.bat diff --git a/windows-installer/template.nsi b/windows-installer/assets/template.nsi similarity index 97% rename from windows-installer/template.nsi rename to windows-installer/assets/template.nsi index 64bceb065..5c551729a 100644 --- a/windows-installer/template.nsi +++ b/windows-installer/assets/template.nsi @@ -1,7 +1,7 @@ -; This NSIS template is based on the built-in one in pynsist 2.6. +; This NSIS template is based on the built-in one in pynsist 2.7. ; Added lines are enclosed within "CERTBOT CUSTOM BEGIN/END" comments. ; If pynsist is upgraded, this template must be updated if necessary using the new built-in one. -; Original file can be found here: https://github.com/takluyver/pynsist/blob/2.6/nsist/pyapp.nsi +; Original file can be found here: https://github.com/takluyver/pynsist/blob/2.7/nsist/pyapp.nsi !define PRODUCT_NAME "[[ib.appname]]" !define PRODUCT_VERSION "[[ib.version]]" @@ -83,6 +83,11 @@ SectionEnd [% block sections %] Section "!${PRODUCT_NAME}" sec_app + ; CERTBOT CUSTOM BEGIN + ; Try to cleanup Certbot pkg directory to avoid dependencies conflicts + RMDir /r "$INSTDIR\pkgs" + ; CERTBOT CUSTOM END + SetRegView [[ib.py_bitness]] SectionIn RO File ${PRODUCT_ICON} diff --git a/windows-installer/setup.py b/windows-installer/setup.py new file mode 100644 index 000000000..cddc9ea18 --- /dev/null +++ b/windows-installer/setup.py @@ -0,0 +1,45 @@ +from setuptools import find_packages +from setuptools import setup + +version = '1.0' + +setup( + name='windows-installer', + version=version, + description='Environment to build the Certbot Windows installer', + url='https://github.com/letsencrypt/letsencrypt', + author="Certbot Project", + author_email='certbot-dev@eff.org', + license='Apache License 2.0', + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Operating System :: Microsoft :: Windows', + 'Topic :: Software Development :: Build Tools', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=[ + # pynsist is pinned to an exact version so we can update + # assets/template.nsi as needed. The file is based on the default + # pynsist NSIS template and pynsist's documentation warns that custom + # templates may need to be updated for them to work with new versions + # of pynsist. See + # https://pynsist.readthedocs.io/en/latest/cfgfile.html#build-section. + 'pynsist==2.7' + ], + entry_points={ + 'console_scripts': [ + 'construct-windows-installer = windows_installer.construct:main', + ], + }, +) diff --git a/windows-installer/windows_installer/__init__.py b/windows-installer/windows_installer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/windows-installer/construct.py b/windows-installer/windows_installer/construct.py similarity index 56% rename from windows-installer/construct.py rename to windows-installer/windows_installer/construct.py index eb199a7e1..91df6b714 100644 --- a/windows-installer/construct.py +++ b/windows-installer/windows_installer/construct.py @@ -1,22 +1,33 @@ #!/usr/bin/env python3 -import contextlib import ctypes import os -import re import shutil import struct import subprocess import sys -import tempfile import time -PYTHON_VERSION = (3, 8, 8) +PYTHON_VERSION = (3, 8, 9) PYTHON_BITNESS = 32 -PYWIN32_VERSION = 300 # do not forget to edit pywin32 dependency accordingly in setup.py NSIS_VERSION = '3.06.1' def main(): + if os.name != 'nt': + raise RuntimeError('This script must be run under Windows.') + + if ctypes.windll.shell32.IsUserAnAdmin() == 0: + # Administrator privileges are required to properly install NSIS through Chocolatey + raise RuntimeError('This script must be run with administrator privileges.') + + if sys.version_info[:2] != PYTHON_VERSION[:2]: + raise RuntimeError('This script must be run with Python {0}' + .format('.'.join(str(item) for item in PYTHON_VERSION[0:2]))) + + if struct.calcsize('P') * 8 != PYTHON_BITNESS: + raise RuntimeError('This script must be run with a {0} bit version of Python.' + .format(PYTHON_BITNESS)) + build_path, repo_path, venv_path, venv_python = _prepare_environment() _copy_assets(build_path, repo_path) @@ -25,14 +36,14 @@ def main(): _prepare_build_tools(venv_path, venv_python, repo_path) _compile_wheels(repo_path, build_path, venv_python) - _build_installer(installer_cfg_path, venv_path) + _build_installer(installer_cfg_path) print('Done') -def _build_installer(installer_cfg_path, venv_path): +def _build_installer(installer_cfg_path): print('Build the installer') - subprocess.check_call([os.path.join(venv_path, 'Scripts', 'pynsist.exe'), installer_cfg_path]) + subprocess.check_call([sys.executable, '-m', 'nsist', installer_cfg_path]) def _compile_wheels(repo_path, build_path, venv_python): @@ -46,69 +57,31 @@ def _compile_wheels(repo_path, build_path, venv_python): # certbot_packages.extend([name for name in os.listdir(repo_path) if name.startswith('certbot-dns-')]) wheels_project = [os.path.join(repo_path, package) for package in certbot_packages] - with _prepare_constraints(repo_path) as constraints_file_path: - env = os.environ.copy() - env['PIP_CONSTRAINT'] = constraints_file_path - command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path] - command.extend(wheels_project) - subprocess.check_call(command, env=env) - - # Cryptography uses now a unique wheel name "cryptography-VERSION-cpXX-abi3-win32.whl where - # cpXX is the lowest supported version of Python (eg. cp36 says that the wheel is compatible - # with Python 3.6+). While technically valid to describe a wheel compliant with the Stable - # Application Binary Interface, this naming convention makes pynsist falsely think that the - # wheel is compatible with Python 3.6 only. - # Let's trick pynsist by renaming the wheel until this is fixed upstream. - for file in os.listdir(wheels_path): - # Given that our Python version is 3.8, this rename files like - # cryptography-VERSION-cpXX-abi3-win32.whl into cryptography-VERSION-cp38-abi3-win32.whl - renamed = re.sub(r'^(.*)-cp\d+-abi3-(\w+)\.whl$', r'\1-cp{0}{1}-abi3-\2.whl' - .format(PYTHON_VERSION[0], PYTHON_VERSION[1]), file) - print(renamed) - if renamed != file: - os.replace(os.path.join(wheels_path, file), os.path.join(wheels_path, renamed)) + constraints_file_path = os.path.join(repo_path, 'tools', 'requirements.txt') + env = os.environ.copy() + env['PIP_CONSTRAINT'] = constraints_file_path + command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path] + command.extend(wheels_project) + subprocess.check_call(command, env=env) def _prepare_build_tools(venv_path, venv_python, repo_path): print('Prepare build tools') subprocess.check_call([sys.executable, '-m', 'venv', venv_path]) subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pipstrap.py')]) - subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pip_install.py'), 'pynsist']) subprocess.check_call(['choco', 'upgrade', '--allow-downgrade', '-y', 'nsis', '--version', NSIS_VERSION]) -@contextlib.contextmanager -def _prepare_constraints(repo_path): - reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_constraints.txt') - reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt') - constraints_certbot = subprocess.check_output( - [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot], - universal_newlines=True) - constraints_pipstrap = subprocess.check_output( - [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_pipstrap], - universal_newlines=True) - workdir = tempfile.mkdtemp() - try: - constraints_file_path = os.path.join(workdir, 'constraints.txt') - with open(constraints_file_path, 'a') as file_h: - file_h.write(constraints_pipstrap) - file_h.write(constraints_certbot) - file_h.write('pywin32=={0}'.format(PYWIN32_VERSION)) - yield constraints_file_path - finally: - shutil.rmtree(workdir) - - def _copy_assets(build_path, repo_path): print('Copy assets') if os.path.exists(build_path): os.rename(build_path, '{0}.{1}.bak'.format(build_path, int(time.time()))) os.makedirs(build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'certbot.ico'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'run.bat'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'template.nsi'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'renew-up.ps1'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'renew-down.ps1'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'certbot.ico'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'run.bat'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'template.nsi'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'renew-up.ps1'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'renew-down.ps1'), build_path) def _generate_pynsist_config(repo_path, build_path): @@ -164,7 +137,7 @@ def _prepare_environment(): raise RuntimeError('Error: Chocolatey (https://chocolatey.org/) needs ' 'to be installed to run this script.') script_path = os.path.realpath(__file__) - repo_path = os.path.dirname(os.path.dirname(script_path)) + repo_path = os.path.dirname(os.path.dirname(os.path.dirname(script_path))) build_path = os.path.join(repo_path, 'windows-installer', 'build') venv_path = os.path.join(build_path, 'venv-config') venv_python = os.path.join(venv_path, 'Scripts', 'python.exe') @@ -173,18 +146,4 @@ def _prepare_environment(): if __name__ == '__main__': - if os.name != 'nt': - raise RuntimeError('This script must be run under Windows.') - - if ctypes.windll.shell32.IsUserAnAdmin() == 0: - # Administrator privileges are required to properly install NSIS through Chocolatey - raise RuntimeError('This script must be run with administrator privileges.') - - if sys.version_info[:2] != PYTHON_VERSION[:2]: - raise RuntimeError('This script must be run with Python {0}' - .format('.'.join(str(item) for item in PYTHON_VERSION[0:2]))) - - if struct.calcsize('P') * 8 != PYTHON_BITNESS: - raise RuntimeError('This script must be run with a {0} bit version of Python.' - .format(PYTHON_BITNESS)) main()