diff --git a/.azure-pipelines/main.yml b/.azure-pipelines/main.yml index 1975d36db..cae4f799c 100644 --- a/.azure-pipelines/main.yml +++ b/.azure-pipelines/main.yml @@ -5,3 +5,4 @@ pr: jobs: - template: templates/jobs/standard-tests-jobs.yml + diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml index c22f1003f..cffedfcb2 100644 --- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml @@ -4,7 +4,7 @@ jobs: - name: IMAGE_NAME value: ubuntu-18.04 - name: PYTHON_VERSION - value: 3.8 + value: 3.9 - group: certbot-common strategy: matrix: @@ -14,30 +14,31 @@ jobs: linux-py37: PYTHON_VERSION: 3.7 TOXENV: py37 + linux-py38: + PYTHON_VERSION: 3.8 + TOXENV: py38 linux-py37-nopin: PYTHON_VERSION: 3.7 TOXENV: py37 CERTBOT_NO_PIN: 1 + linux-external-mock: + TOXENV: external-mock linux-boulder-v1-integration-certbot-oldest: + PYTHON_VERSION: 3.6 TOXENV: integration-certbot-oldest ACME_SERVER: boulder-v1 linux-boulder-v2-integration-certbot-oldest: + PYTHON_VERSION: 3.6 TOXENV: integration-certbot-oldest ACME_SERVER: boulder-v2 linux-boulder-v1-integration-nginx-oldest: + PYTHON_VERSION: 3.6 TOXENV: integration-nginx-oldest ACME_SERVER: boulder-v1 linux-boulder-v2-integration-nginx-oldest: + PYTHON_VERSION: 3.6 TOXENV: integration-nginx-oldest ACME_SERVER: boulder-v2 - linux-boulder-v1-py27-integration: - PYTHON_VERSION: 2.7 - TOXENV: integration - ACME_SERVER: boulder-v1 - linux-boulder-v2-py27-integration: - PYTHON_VERSION: 2.7 - TOXENV: integration - ACME_SERVER: boulder-v2 linux-boulder-v1-py36-integration: PYTHON_VERSION: 3.6 TOXENV: integration @@ -62,10 +63,20 @@ jobs: PYTHON_VERSION: 3.8 TOXENV: integration ACME_SERVER: boulder-v2 + linux-boulder-v1-py39-integration: + PYTHON_VERSION: 3.9 + TOXENV: integration + ACME_SERVER: boulder-v1 + linux-boulder-v2-py39-integration: + PYTHON_VERSION: 3.9 + TOXENV: integration + ACME_SERVER: boulder-v2 nginx-compat: TOXENV: nginx_compat - le-auto-oraclelinux6: - TOXENV: le_auto_oraclelinux6 + linux-integration-rfc2136: + IMAGE_NAME: ubuntu-18.04 + PYTHON_VERSION: 3.8 + TOXENV: integration-dns-rfc2136 docker-dev: TOXENV: docker_dev macos-farmtest-apache2: diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index f0c6b6e49..28255919f 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -56,7 +56,7 @@ jobs: steps: - task: UsePythonVersion@0 inputs: - versionSpec: 3.7 + versionSpec: 3.8 architecture: x86 addToPath: true - script: python windows-installer/construct.py @@ -144,7 +144,7 @@ jobs: git config --global user.name "$(Build.RequestedFor)" mkdir -p ~/.local/share/snapcraft/provider/launchpad cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials - python3 tools/snap/build_remote.py ALL --archs ${ARCHS} + python3 tools/snap/build_remote.py ALL --archs ${ARCHS} --timeout 19800 displayName: Build snaps - script: | set -e diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index 69e8b279b..fec11c6c5 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -1,47 +1,45 @@ jobs: - job: test variables: - PYTHON_VERSION: 3.8 + PYTHON_VERSION: 3.9 strategy: matrix: - macos-py27: + macos-py36: IMAGE_NAME: macOS-10.15 - PYTHON_VERSION: 2.7 - TOXENV: py27 - macos-py38: + PYTHON_VERSION: 3.6 + TOXENV: py36 + macos-py39: IMAGE_NAME: macOS-10.15 - PYTHON_VERSION: 3.8 - TOXENV: py38 + PYTHON_VERSION: 3.9 + TOXENV: py39 windows-py36: IMAGE_NAME: vs2017-win2016 PYTHON_VERSION: 3.6 TOXENV: py36 - windows-py37-cover: + windows-py38-cover: IMAGE_NAME: vs2017-win2016 - PYTHON_VERSION: 3.7 - TOXENV: py37-cover + PYTHON_VERSION: 3.8 + TOXENV: py38-cover windows-integration-certbot: IMAGE_NAME: vs2017-win2016 - PYTHON_VERSION: 3.7 + PYTHON_VERSION: 3.8 TOXENV: integration-certbot linux-oldest-tests-1: IMAGE_NAME: ubuntu-18.04 - TOXENV: py27-{acme,apache,apache-v2,certbot}-oldest + PYTHON_VERSION: 3.6 + TOXENV: '{acme,apache,apache-v2,certbot}-oldest' linux-oldest-tests-2: IMAGE_NAME: ubuntu-18.04 - TOXENV: py27-{dns,nginx}-oldest - linux-py27: - IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 2.7 - TOXENV: py27 + PYTHON_VERSION: 3.6 + TOXENV: '{dns,nginx}-oldest' linux-py36: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.6 TOXENV: py36 - linux-py38-cover: + linux-py39-cover: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 3.8 - TOXENV: py38-cover + PYTHON_VERSION: 3.9 + TOXENV: py39-cover linux-py37-lint: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.7 @@ -52,24 +50,29 @@ jobs: TOXENV: mypy linux-integration: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 2.7 + PYTHON_VERSION: 3.8 TOXENV: integration ACME_SERVER: pebble apache-compat: IMAGE_NAME: ubuntu-18.04 TOXENV: apache_compat - le-auto-centos6: + le-modification: IMAGE_NAME: ubuntu-18.04 - TOXENV: le_auto_centos6 + TOXENV: modification apacheconftest: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 2.7 + PYTHON_VERSION: 3.6 TOXENV: apacheconftest-with-pebble nginxroundtrip: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 2.7 + PYTHON_VERSION: 3.6 TOXENV: nginxroundtrip pool: vmImage: $(IMAGE_NAME) steps: - template: ../steps/tox-steps.yml + - job: test_sphinx_builds + pool: + vmImage: ubuntu-20.04 + steps: + - template: ../steps/sphinx-steps.yml diff --git a/.azure-pipelines/templates/stages/notify-failure-stage.yml b/.azure-pipelines/templates/stages/notify-failure-stage.yml index 1542f5ebc..c47342690 100644 --- a/.azure-pipelines/templates/stages/notify-failure-stage.yml +++ b/.azure-pipelines/templates/stages/notify-failure-stage.yml @@ -5,7 +5,7 @@ stages: variables: - group: certbot-common pool: - vmImage: ubuntu-latest + vmImage: ubuntu-20.04 steps: - bash: | set -e diff --git a/.azure-pipelines/templates/steps/sphinx-steps.yml b/.azure-pipelines/templates/steps/sphinx-steps.yml new file mode 100644 index 000000000..23c258bbc --- /dev/null +++ b/.azure-pipelines/templates/steps/sphinx-steps.yml @@ -0,0 +1,23 @@ +steps: + - bash: | + FINAL_STATUS=0 + declare -a FAILED_BUILDS + python3 -m venv .venv + source .venv/bin/activate + python tools/pipstrap.py + for doc_path in */docs + do + echo "" + echo "##[group]Building $doc_path" + pip install -q -e $doc_path/..[docs] + if ! sphinx-build -W --keep-going -b html $doc_path $doc_path/_build/html; then + FINAL_STATUS=1 + FAILED_BUILDS[${#FAILED_BUILDS[@]}]="${doc_path%/docs}" + fi + echo "##[endgroup]" + done + if [[ $FINAL_STATUS -ne 0 ]]; then + echo "##[error]The following builds failed: ${FAILED_BUILDS[*]}" + exit 1 + fi + displayName: Build Sphinx Documentation diff --git a/.azure-pipelines/templates/steps/tox-steps.yml b/.azure-pipelines/templates/steps/tox-steps.yml index a9f78d36b..ecf3d6032 100644 --- a/.azure-pipelines/templates/steps/tox-steps.yml +++ b/.azure-pipelines/templates/steps/tox-steps.yml @@ -45,11 +45,7 @@ steps: export TARGET_BRANCH="`echo "${BUILD_SOURCEBRANCH}" | sed -E 's!refs/(heads|tags)/!!g'`" [ -z "${SYSTEM_PULLREQUEST_TARGETBRANCH}" ] || export TARGET_BRANCH="${SYSTEM_PULLREQUEST_TARGETBRANCH}" env - if [[ "${TOXENV}" == *"oldest"* ]]; then - tools/run_oldest_tests.sh - else - python -m tox - fi + python -m tox env: AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) diff --git a/.dockerignore b/.dockerignore index b94bf7960..2ce8a8209 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,5 +8,4 @@ .git .tox venv -venv3 docs diff --git a/.envrc b/.envrc index 4d2077ebb..43c3170d6 100644 --- a/.envrc +++ b/.envrc @@ -3,7 +3,7 @@ # activated and then deactivated when you cd elsewhere. Developers have to have # direnv set up and run `direnv allow` to allow this file to execute on their # system. You can find more information at https://direnv.net/. -. venv3/bin/activate +. venv/bin/activate # direnv doesn't support modifying PS1 so we unset it to squelch the error # it'll otherwise print about this being done in the activate script. See # https://github.com/direnv/direnv/wiki/PS1. If you would like your shell diff --git a/.pylintrc b/.pylintrc index 0e78828bd..a2468b0cf 100644 --- a/.pylintrc +++ b/.pylintrc @@ -254,7 +254,7 @@ ignore-mixin-members=yes # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis -ignored-modules=pkg_resources,confargparse,argparse,six.moves,six.moves.urllib +ignored-modules=pkg_resources,confargparse,argparse # import errors ignored only in 1.4.4 # https://bitbucket.org/logilab/pylint/commits/cd000904c9e2 diff --git a/AUTHORS.md b/AUTHORS.md index 56fe2a3d8..cb60bfd87 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -1,6 +1,7 @@ Authors ======= +* [Aaron Gable](https://github.com/aarongable) * [Aaron Zirbes](https://github.com/aaronzirbes) * Aaron Zuehlke * Ada Lovelace @@ -60,6 +61,7 @@ Authors * [DanCld](https://github.com/DanCld) * [Daniel Albers](https://github.com/AID) * [Daniel Aleksandersen](https://github.com/da2x) +* [Daniel Almasi](https://github.com/almasen) * [Daniel Convissor](https://github.com/convissor) * [Daniel "Drex" Drexler](https://github.com/aeturnum) * [Daniel Huang](https://github.com/dhuang) @@ -149,11 +151,13 @@ Authors * [Lior Sabag](https://github.com/liorsbg) * [Lipis](https://github.com/lipis) * [lord63](https://github.com/lord63) +* [Lorenzo Fundaró](https://github.com/lfundaro) * [Luca Beltrame](https://github.com/lbeltrame) * [Luca Ebach](https://github.com/lucebac) * [Luca Olivetti](https://github.com/olivluca) * [Luke Rogers](https://github.com/lukeroge) * [Maarten](https://github.com/mrtndwrd) +* [Mads Jensen](https://github.com/atombrella) * [Maikel Martens](https://github.com/krukas) * [Malte Janduda](https://github.com/MalteJ) * [Mantas Mikulėnas](https://github.com/grawity) @@ -213,6 +217,7 @@ Authors * [Richard Barnes](https://github.com/r-barnes) * [Richard Panek](https://github.com/kernelpanek) * [Robert Buchholz](https://github.com/rbu) +* [Robert Dailey](https://github.com/pahrohfit) * [Robert Habermann](https://github.com/frennkie) * [Robert Xiao](https://github.com/nneonneo) * [Roland Shoemaker](https://github.com/rolandshoemaker) diff --git a/Dockerfile-dev b/Dockerfile-dev index ae197b1cb..86847f8fd 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -15,6 +15,6 @@ RUN apt-get update && \ /tmp/* \ /var/tmp/* -RUN VENV_NAME="../venv3" python3 tools/venv3.py +RUN VENV_NAME="../venv" python3 tools/venv.py -ENV PATH /opt/certbot/venv3/bin:$PATH +ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index d1679fcad..8b6ce88c0 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -6,7 +6,6 @@ This module is an implementation of the `ACME protocol`_. """ import sys -import warnings # This code exists to keep backwards compatibility with people using acme.jose # before it became the standalone josepy package. diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index b9c6b7eb2..41a2aa258 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -9,7 +9,6 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose import requests -import six from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from OpenSSL import crypto @@ -145,12 +144,11 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return jobj -@six.add_metaclass(abc.ABCMeta) -class KeyAuthorizationChallenge(_TokenChallenge): +class KeyAuthorizationChallenge(_TokenChallenge, metaclass=abc.ABCMeta): """Challenge based on Key Authorization. :param response_cls: Subclass of `KeyAuthorizationChallengeResponse` - that will be used to generate `response`. + that will be used to generate ``response``. :param str typ: type of the challenge """ typ = NotImplemented diff --git a/acme/acme/client.py b/acme/acme/client.py index d413ce13d..33e515124 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -4,9 +4,9 @@ import collections import datetime from email.utils import parsedate_tz import heapq +import http.client as http_client import logging import re -import sys import time import josepy as jose @@ -15,8 +15,6 @@ import requests from requests.adapters import HTTPAdapter from requests.utils import parse_header_links from requests_toolbelt.adapters.source import SourceAddressAdapter -import six -from six.moves import http_client from acme import crypto_util from acme import errors @@ -30,23 +28,12 @@ from acme.mixins import VersionedLEACMEMixin logger = logging.getLogger(__name__) -# Prior to Python 2.7.9 the stdlib SSL module did not allow a user to configure -# many important security related options. On these platforms we use PyOpenSSL -# for SSL, which does allow these options to be configured. -# https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning -if sys.version_info < (2, 7, 9): # pragma: no cover - try: - requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() # type: ignore - except AttributeError: - import urllib3.contrib.pyopenssl - urllib3.contrib.pyopenssl.inject_into_urllib3() - DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class ClientBase(object): +class ClientBase: """ACME client base object. :ivar messages.Directory directory: @@ -260,7 +247,7 @@ class Client(ClientBase): if net is None: net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) - if isinstance(directory, six.string_types): + if isinstance(directory, str): directory = messages.Directory.from_json( net.get(directory).json()) super(Client, self).__init__(directory=directory, @@ -475,7 +462,7 @@ class Client(ClientBase): exhausted.add(authzr) if exhausted or any(authzr.body.status == messages.STATUS_INVALID - for authzr in six.itervalues(updated)): + for authzr in updated.values()): raise errors.PollError(exhausted, updated) updated_authzrs = tuple(updated[authzr] for authzr in authzrs) @@ -808,7 +795,7 @@ class ClientV2(ClientBase): if 'rel' in l and 'url' in l and l['rel'] == relation_type] -class BackwardsCompatibleClientV2(object): +class BackwardsCompatibleClientV2: """ACME client wrapper that tends towards V2-style calls, but supports V1 servers. @@ -951,7 +938,7 @@ class BackwardsCompatibleClientV2(object): return self.client.external_account_required() -class ClientNetwork(object): +class ClientNetwork: """Wrapper around requests that signs POSTs for authentication. Also adds user agent, and handles Content-Type. diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index f8b7e2b30..a14737053 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore -class _DefaultCertSelection(object): +class _DefaultCertSelection: def __init__(self, certs): self.certs = certs @@ -36,7 +36,7 @@ class _DefaultCertSelection(object): return self.certs.get(server_name, None) -class SSLSocket(object): # pylint: disable=too-few-public-methods +class SSLSocket: # pylint: disable=too-few-public-methods """SSL wrapper for sockets. :ivar socket sock: Original wrapped socket. @@ -93,7 +93,7 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) - class FakeConnection(object): + class FakeConnection: """Fake OpenSSL.SSL.Connection.""" # pylint: disable=missing-function-docstring @@ -166,7 +166,7 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu " from {0}:{1}".format( source_address[0], source_address[1] - ) if socket_kwargs else "" + ) if any(source_address) else "" ) socket_tuple = (host, port) # type: Tuple[str, int] sock = socket.create_connection(socket_tuple, **socket_kwargs) # type: ignore @@ -186,6 +186,7 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu raise errors.Error(error) return client_ssl.get_peer_certificate() + def make_csr(private_key_pem, domains, must_staple=False): """Generate a CSR containing a list of domains as subjectAltNames. @@ -217,6 +218,7 @@ def make_csr(private_key_pem, domains, must_staple=False): return crypto.dump_certificate_request( crypto.FILETYPE_PEM, csr) + def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): common_name = loaded_cert_or_req.get_subject().CN sans = _pyopenssl_cert_or_req_san(loaded_cert_or_req) @@ -225,6 +227,7 @@ def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): return sans return [common_name] + [d for d in sans if d != common_name] + def _pyopenssl_cert_or_req_san(cert_or_req): """Get Subject Alternative Names from certificate or CSR using pyOpenSSL. @@ -317,6 +320,7 @@ def gen_ss_cert(key, domains, not_before=None, cert.sign(key, "sha256") return cert + def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 806657940..5ca5a4fa2 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -49,7 +49,7 @@ class MissingNonce(NonceError): Replay-Nonce header field in each successful response to a POST it provides to a client (...)". - :ivar requests.Response response: HTTP Response + :ivar requests.Response ~.response: HTTP Response """ def __init__(self, response, *args, **kwargs): diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 388fc4a58..91308fef6 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -1,16 +1,14 @@ -"""Shim class to not have to depend on typing module in prod.""" -import sys +"""Simple shim around the typing module. + +This was useful when this code supported Python 2 and typing wasn't always +available. This code is being kept for now for backwards compatibility. + +""" +from typing import * # pylint: disable=wildcard-import, unused-wildcard-import +from typing import Collection, IO # type: ignore -class TypingClass(object): +class TypingClass: """Ignore import errors by getting anything""" def __getattr__(self, name): - return None - -try: - # mypy doesn't respect modifying sys.modules - from typing import * # pylint: disable=wildcard-import, unused-wildcard-import - from typing import Collection, IO # type: ignore -except ImportError: - # mypy complains because TypingClass is not a module - sys.modules[__name__] = TypingClass() # type: ignore + return None # pragma: no cover diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 6325ed57f..0d73037ae 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,8 +1,8 @@ """ACME protocol messages.""" import json +from collections.abc import Hashable import josepy as jose -import six from acme import challenges from acme import errors @@ -11,13 +11,6 @@ from acme import jws from acme import util from acme.mixins import ResourceMixin -try: - from collections.abc import Hashable -except ImportError: # pragma: no cover - from collections import Hashable - - - OLD_ERROR_PREFIX = "urn:acme:error:" ERROR_PREFIX = "urn:ietf:params:acme:error:" @@ -68,7 +61,6 @@ def is_acme_error(err): return False -@six.python_2_unicode_compatible class Error(jose.JSONObjectWithFields, errors.Error): """ACME error. @@ -158,9 +150,6 @@ class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore def __hash__(self): return hash((self.__class__, self.name)) - def __ne__(self, other): - return not self == other - class Status(_Constant): """ACME "status" field.""" @@ -275,7 +264,7 @@ class Resource(jose.JSONObjectWithFields): class ResourceWithURI(Resource): """ACME Resource with URI. - :ivar unicode uri: Location of the resource. + :ivar unicode ~.uri: Location of the resource. """ uri = jose.Field('uri') # no ChallengeResource.uri @@ -285,7 +274,7 @@ class ResourceBody(jose.JSONObjectWithFields): """ACME Resource Body.""" -class ExternalAccountBinding(object): +class ExternalAccountBinding: """ACME External Account Binding""" @classmethod @@ -627,7 +616,7 @@ class Order(ResourceBody): :ivar str finalize: URL to POST to to request issuance once all authorizations have "valid" status. :ivar datetime.datetime expires: When the order expires. - :ivar .Error error: Any error that occurred during finalization, if applicable. + :ivar ~.Error error: Any error that occurred during finalization, if applicable. """ identifiers = jose.Field('identifiers', omitempty=True) status = jose.Field('status', decoder=Status.from_json, diff --git a/acme/acme/mixins.py b/acme/acme/mixins.py index 1cd050ccc..0e1e0977c 100644 --- a/acme/acme/mixins.py +++ b/acme/acme/mixins.py @@ -1,7 +1,7 @@ """Useful mixins for Challenge and Resource objects""" -class VersionedLEACMEMixin(object): +class VersionedLEACMEMixin: """This mixin stores the version of Let's Encrypt's endpoint being used.""" @property def le_acme_version(self): diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 7a61ba868..f5bc548b6 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -1,14 +1,13 @@ """Support for standalone client challenge solvers. """ import collections import functools +import http.client as http_client +import http.server as BaseHTTPServer import logging import socket +import socketserver import threading -from six.moves import BaseHTTPServer # type: ignore -from six.moves import http_client -from six.moves import socketserver # type: ignore - from acme import challenges from acme import crypto_util from acme.magic_typing import List @@ -54,7 +53,7 @@ class ACMEServerMixin: allow_reuse_address = True -class BaseDualNetworkedServers(object): +class BaseDualNetworkedServers: """Base class for a pair of IPv6 and IPv4 servers that tries to do everything it's asked for both servers, but where failures in one server don't affect the other. diff --git a/acme/acme/util.py b/acme/acme/util.py index b3b0d79eb..20fa455fd 100644 --- a/acme/acme/util.py +++ b/acme/acme/util.py @@ -1,7 +1,6 @@ """ACME utilities.""" -import six def map_keys(dikt, func): """Map dictionary keys.""" - return {func(key): value for key, value in six.iteritems(dikt)} + return {func(key): value for key, value in dikt.items()} diff --git a/acme/docs/conf.py b/acme/docs/conf.py index d3e7be371..d419326df 100644 --- a/acme/docs/conf.py +++ b/acme/docs/conf.py @@ -85,7 +85,9 @@ language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = [ + '_build', +] # The reST default role (used for this markup: `text`) to use for all # documents. diff --git a/acme/docs/man/jws.rst b/acme/docs/man/jws.rst index d7ff8f405..7e83328b2 100644 --- a/acme/docs/man/jws.rst +++ b/acme/docs/man/jws.rst @@ -1 +1,3 @@ +:orphan: + .. literalinclude:: ../jws-help.txt diff --git a/acme/setup.py b/acme/setup.py index 96adc2963..90ab5c9f3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,40 +1,25 @@ -from distutils.version import LooseVersion import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - # load_pem_private/public_key (>=0.6) - # rsa_recover_prime_factors (>=0.8) - 'cryptography>=1.2.3', + 'cryptography>=2.1.4', # formerly known as acme.jose: # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - # Connection.set_tlsext_host_name (>=0.13) + matching Xenial requirements (>=0.15.1) - 'PyOpenSSL>=0.15.1', + 'PyOpenSSL>=17.3.0', 'pyrfc3339', 'pytz', - 'requests[security]>=2.6.0', # security extras added in 2.4.1 + 'requests>=2.6.0', 'requests-toolbelt>=0.3.0', - 'setuptools', - 'six>=1.9.0', # needed for python_2_unicode_compatible + 'setuptools>=39.0.1', ] -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - dev_extras = [ 'pytest', 'pytest-xdist', @@ -54,18 +39,17 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index 70371051c..cc604b0de 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -1,14 +1,11 @@ """Tests for acme.challenges.""" +import urllib.parse as urllib_parse import unittest +from unittest import mock import josepy as jose import OpenSSL -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore import requests -from six.moves.urllib import parse as urllib_parse from acme import errors diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index c84878c42..6f9aecda2 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -2,17 +2,14 @@ # pylint: disable=too-many-lines import copy import datetime +import http.client as http_client import json import unittest +from unittest import mock import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore import OpenSSL import requests -from six.moves import http_client # pylint: disable=import-error from acme import challenges from acme import errors diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index 705a3c856..f271ad37d 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -1,14 +1,13 @@ """Tests for acme.crypto_util.""" import itertools import socket +import socketserver import threading import time import unittest import josepy as jose import OpenSSL -import six -from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import errors import test_util @@ -27,8 +26,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): class _TestServer(socketserver.TCPServer): - # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init - def server_bind(self): # pylint: disable=missing-docstring self.socket = SSLSocket(socket.socket(), certs) @@ -62,7 +59,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): self.assertRaises(errors.Error, self._probe, b'bar') def test_probe_connection_error(self): - # pylint has a hard time with six self.server.server_close() original_timeout = socket.getdefaulttimeout() try: @@ -121,9 +117,9 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): @classmethod def _get_idn_names(cls): """Returns expected names from '{cert,csr}-idnsans.pem'.""" - chars = [six.unichr(i) for i in itertools.chain(range(0x3c3, 0x400), - range(0x641, 0x6fc), - range(0x1820, 0x1877))] + chars = [chr(i) for i in itertools.chain(range(0x3c3, 0x400), + range(0x641, 0x6fc), + range(0x1820, 0x1877))] return [''.join(chars[i: i + 45]) + '.invalid' for i in range(0, len(chars), 45)] diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index fb90a3f0d..11c57059c 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -1,10 +1,6 @@ """Tests for acme.errors.""" import unittest - -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore +from unittest import mock class BadNonceTest(unittest.TestCase): diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 9e4fd29f5..257164d77 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -1,11 +1,7 @@ """Tests for acme.magic_typing.""" import sys import unittest - -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore +from unittest import mock class MagicTypingTest(unittest.TestCase): @@ -26,19 +22,6 @@ class MagicTypingTest(unittest.TestCase): del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing - def test_import_failure(self): - try: - import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover - sys.modules['typing'] = None - if 'acme.magic_typing' in sys.modules: - del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text - self.assertTrue(Text is None) - del sys.modules['acme.magic_typing'] - sys.modules['typing'] = temp_typing - if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index 3458105b2..74d1737ec 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -1,11 +1,8 @@ """Tests for acme.messages.""" import unittest +from unittest import mock import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore from acme import challenges import test_util @@ -108,11 +105,11 @@ class ConstantTest(unittest.TestCase): def test_equality(self): const_a_prime = self.MockConstant('a') - self.assertFalse(self.const_a == self.const_b) - self.assertTrue(self.const_a == const_a_prime) + self.assertNotEqual(self.const_a, self.const_b) + self.assertEqual(self.const_a, const_a_prime) - self.assertTrue(self.const_a != self.const_b) - self.assertFalse(self.const_a != const_a_prime) + self.assertNotEqual(self.const_a, self.const_b) + self.assertEqual(self.const_a, const_a_prime) class DirectoryTest(unittest.TestCase): diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 3d068fb46..e6aa8f2d6 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -1,16 +1,13 @@ """Tests for acme.standalone.""" +import http.client as http_client import socket +import socketserver import threading import unittest +from unittest import mock import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore import requests -from six.moves import http_client # pylint: disable=import-error -from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import challenges from acme import crypto_util diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index c20a4fdd6..16def1998 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -327,6 +327,9 @@ class ApacheConfigurator(common.Installer): if self.version < (2, 2): raise errors.NotSupportedError( "Apache Version {0} not supported.".format(str(self.version))) + elif self.version < (2, 4): + logger.warning('Support for Apache 2.2 is deprecated and will be removed in a ' + 'future release.') # Recover from previous crash before Augeas initialization to have the # correct parse tree from the get go. diff --git a/certbot-apache/certbot_apache/_internal/dualparser.py b/certbot-apache/certbot_apache/_internal/dualparser.py index eef8f2a0e..1ba23e92f 100644 --- a/certbot-apache/certbot_apache/_internal/dualparser.py +++ b/certbot-apache/certbot_apache/_internal/dualparser.py @@ -4,7 +4,7 @@ from certbot_apache._internal import augeasparser from certbot_apache._internal import apacheparser -class DualNodeBase(object): +class DualNodeBase: """ Dual parser interface for in development testing. This is used as the base class for dual parser interface classes. This class handles runtime attribute value assertions.""" diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 647790c41..77daa5b56 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -100,12 +100,10 @@ For this reason the internal representation of data should not ignore the case. """ import abc -import six -@six.add_metaclass(abc.ABCMeta) -class ParserNode(object): +class ParserNode(object, metaclass=abc.ABCMeta): """ ParserNode is the basic building block of the tree of such nodes, representing the structure of the configuration. It is largely meant to keep @@ -204,9 +202,7 @@ class ParserNode(object): """ -# Linter rule exclusion done because of https://github.com/PyCQA/pylint/issues/179 -@six.add_metaclass(abc.ABCMeta) # pylint: disable=abstract-method -class CommentNode(ParserNode): +class CommentNode(ParserNode, metaclass=abc.ABCMeta): """ CommentNode class is used for representation of comments within the parsed configuration structure. Because of the nature of comments, it is not able @@ -249,8 +245,7 @@ class CommentNode(ParserNode): metadata=kwargs.get('metadata', {})) # pragma: no cover -@six.add_metaclass(abc.ABCMeta) -class DirectiveNode(ParserNode): +class DirectiveNode(ParserNode, metaclass=abc.ABCMeta): """ DirectiveNode class represents a configuration directive within the configuration. It can have zero or more parameters attached to it. Because of the nature of @@ -325,8 +320,7 @@ class DirectiveNode(ParserNode): """ -@six.add_metaclass(abc.ABCMeta) -class BlockNode(DirectiveNode): +class BlockNode(DirectiveNode, metaclass=abc.ABCMeta): """ BlockNode class represents a block of nested configuration directives, comments and other blocks as its children. A BlockNode can have zero or more parameters diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 498766744..e2fe48cf8 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -20,9 +20,6 @@ class Addr(common.Addr): self.is_wildcard() and other.is_wildcard())) return False - def __ne__(self, other): - return not self.__eq__(other) - def __repr__(self): return "certbot_apache._internal.obj.Addr(" + repr(self.tup) + ")" @@ -98,7 +95,7 @@ class Addr(common.Addr): return self.get_addr_obj(port) -class VirtualHost(object): +class VirtualHost: """Represents an Apache Virtualhost. :ivar str filep: file path of VH @@ -191,9 +188,6 @@ class VirtualHost(object): return False - def __ne__(self, other): - return not self.__eq__(other) - def __hash__(self): return hash((self.filep, self.path, tuple(self.addrs), tuple(self.get_names()), diff --git a/certbot-apache/certbot_apache/_internal/override_suse.py b/certbot-apache/certbot_apache/_internal/override_suse.py index eee9b0b64..afce98dfa 100644 --- a/certbot-apache/certbot_apache/_internal/override_suse.py +++ b/certbot-apache/certbot_apache/_internal/override_suse.py @@ -14,10 +14,10 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", logs_root="/var/log/apache2", - ctl="apache2ctl", - version_cmd=['apache2ctl', '-v'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], + ctl="apachectl", + version_cmd=['apachectl', '-v'], + restart_cmd=['apachectl', 'graceful'], + conftest_cmd=['apachectl', 'configtest'], enmod="a2enmod", dismod="a2dismod", le_vhost_ext="-le-ssl.conf", diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 4d997545b..a1ea02b18 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -3,9 +3,6 @@ import copy import fnmatch import logging import re -import sys - -import six from acme.magic_typing import Dict from acme.magic_typing import List @@ -17,7 +14,7 @@ from certbot_apache._internal import constants logger = logging.getLogger(__name__) -class ApacheParser(object): +class ApacheParser: """Class handles the fine details of parsing the Apache Configuration. .. todo:: Make parsing general... remove sites-available etc... @@ -275,7 +272,7 @@ class ApacheParser(object): while len(mods) != prev_size: prev_size = len(mods) - for match_name, match_filename in six.moves.zip( + for match_name, match_filename in zip( iterator, iterator): mod_name = self.get_arg(match_name) mod_filename = self.get_arg(match_filename) @@ -738,9 +735,6 @@ class ApacheParser(object): :rtype: str """ - if sys.version_info < (3, 6): - # This strips off final /Z(?ms) - return fnmatch.translate(clean_fn_match)[:-7] # pragma: no cover # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z return fnmatch.translate(clean_fn_match)[4:-3] # pragma: no cover diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index cd1a24f93..43165a9dd 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,11 +1,7 @@ -from distutils.version import LooseVersion -import sys - -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -13,20 +9,11 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.6.0', 'python-augeas', - 'setuptools', + 'setuptools>=39.0.1', 'zope.component', 'zope.interface', ] -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - dev_extras = [ 'apacheconfig>=0.3.2', ] @@ -39,7 +26,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -47,12 +34,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index d15600215..db1c46b52 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -7,7 +7,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six # pylint: disable=unused-import # six is used in mock.patch() from certbot import errors from certbot_apache._internal import constants diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 38d07aefe..302bf0023 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -10,7 +10,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six # pylint: disable=unused-import # six is used in mock.patch() from acme import challenges from certbot import achallenges @@ -726,7 +725,7 @@ class MultipleVhostsTest(util.ApacheTest): # This calls open self.config.reverter.register_file_creation = mock.Mock() mock_open.side_effect = IOError - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertRaises( errors.PluginError, self.config.make_vhost_ssl, self.vh_truth[0]) @@ -1350,10 +1349,10 @@ class MultipleVhostsTest(util.ApacheTest): # And the actual returned values self.assertEqual(len(vhs), 1) - self.assertTrue(vhs[0].name == "certbot.demo") + self.assertEqual(vhs[0].name, "certbot.demo") self.assertTrue(vhs[0].ssl) - self.assertFalse(vhs[0] == self.vh_truth[3]) + self.assertNotEqual(vhs[0], self.vh_truth[3]) @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.make_vhost_ssl") def test_choose_vhosts_wildcard_no_ssl(self, mock_makessl): @@ -1464,10 +1463,10 @@ class MultipleVhostsTest(util.ApacheTest): self.config.parser.aug.match = mock_match vhs = self.config.get_virtual_hosts() self.assertEqual(len(vhs), 2) - self.assertTrue(vhs[0] == self.vh_truth[1]) + self.assertEqual(vhs[0], self.vh_truth[1]) # mock_vhost should have replaced the vh_truth[0], because its filepath # isn't a symlink - self.assertTrue(vhs[1] == mock_vhost) + self.assertEqual(vhs[1], mock_vhost) class AugeasVhostsTest(util.ApacheTest): @@ -1834,7 +1833,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): def test_open_module_file(self): mock_open = mock.mock_open(read_data="testing 12 3") - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertEqual(self.config._open_module_file("/nonsense/"), "testing 12 3") if __name__ == "__main__": diff --git a/certbot-apache/tests/dualnode_test.py b/certbot-apache/tests/dualnode_test.py index 44cc69ff4..ef7f8b2d8 100644 --- a/certbot-apache/tests/dualnode_test.py +++ b/certbot-apache/tests/dualnode_test.py @@ -412,9 +412,9 @@ class DualParserNodeTest(unittest.TestCase): # pylint: disable=too-many-public- ancestor=self.block, filepath="/path/to/whatever", metadata=self.metadata) - self.assertFalse(self.block == ne_block) - self.assertFalse(self.directive == ne_directive) - self.assertFalse(self.comment == ne_comment) + self.assertNotEqual(self.block, ne_block) + self.assertNotEqual(self.directive, ne_directive) + self.assertNotEqual(self.comment, ne_comment) def test_parsed_paths(self): mock_p = mock.MagicMock(return_value=['/path/file.conf', diff --git a/certbot-apache/tests/obj_test.py b/certbot-apache/tests/obj_test.py index eac6a64ef..8aae3ad45 100644 --- a/certbot-apache/tests/obj_test.py +++ b/certbot-apache/tests/obj_test.py @@ -27,14 +27,14 @@ class VirtualHostTest(unittest.TestCase): "certbot_apache._internal.obj.Addr(('127.0.0.1', '443'))") def test_eq(self): - self.assertTrue(self.vhost1b == self.vhost1) - self.assertFalse(self.vhost1 == self.vhost2) + self.assertEqual(self.vhost1b, self.vhost1) + self.assertNotEqual(self.vhost1, self.vhost2) self.assertEqual(str(self.vhost1b), str(self.vhost1)) - self.assertFalse(self.vhost1b == 1234) + self.assertNotEqual(self.vhost1b, 1234) def test_ne(self): - self.assertTrue(self.vhost1 != self.vhost2) - self.assertFalse(self.vhost1 != self.vhost1b) + self.assertNotEqual(self.vhost1, self.vhost2) + self.assertEqual(self.vhost1, self.vhost1b) def test_conflicts(self): from certbot_apache._internal.obj import Addr @@ -128,13 +128,13 @@ class AddrTest(unittest.TestCase): self.assertTrue(self.addr1.conflicts(self.addr2)) def test_equal(self): - self.assertTrue(self.addr1 == self.addr2) - self.assertFalse(self.addr == self.addr1) - self.assertFalse(self.addr == 123) + self.assertEqual(self.addr1, self.addr2) + self.assertNotEqual(self.addr, self.addr1) + self.assertNotEqual(self.addr, 123) def test_not_equal(self): - self.assertFalse(self.addr1 != self.addr2) - self.assertTrue(self.addr != self.addr1) + self.assertEqual(self.addr1, self.addr2) + self.assertNotEqual(self.addr, self.addr1) if __name__ == "__main__": diff --git a/certbot-auto b/certbot-auto index 2a0cda9b3..24af0b4c3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.9.0" +LE_AUTO_VERSION="1.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -799,15 +799,14 @@ BootstrapMageiaCommon() { # that function. If Bootstrap is set to a function that doesn't install any # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then - Bootstrap() { - BootstrapMessage "Debian-based OSes" - BootstrapDebCommon - } - BOOTSTRAP_VERSION="BootstrapDebCommon $BOOTSTRAP_DEB_COMMON_VERSION" + DEPRECATED_OS=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then + DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -840,12 +839,7 @@ elif [ -f /etc/redhat-release ]; then INTERACTIVE_BOOTSTRAP=1 fi - Bootstrap() { - BootstrapMessage "Legacy RedHat-based OSes that will use Python3" - BootstrapRpmPython3Legacy - } USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION" # Try now to enable SCL rh-python36 for systems already bootstrapped # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto @@ -864,43 +858,38 @@ elif [ -f /etc/redhat-release ]; then fi if [ "$RPM_USE_PYTHON_3" = 1 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/arch-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/manjaro-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/gentoo-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif uname | grep -iq FreeBSD ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif uname | grep -iq Darwin ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then - Bootstrap() { - ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 else DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 fi # We handle this case after determining the normal bootstrap version to allow @@ -1129,7 +1118,9 @@ if [ "$1" = "--le-auto-phase2" ]; then fi if [ -f "$VENV_BIN/letsencrypt" -a "$INSTALL_ONLY" != 1 ]; then - error "Certbot will no longer receive updates." + error "certbot-auto and its Certbot installation will no longer receive updates." + error "You will not receive any bug fixes including those fixing server compatibility" + error "or security problems." error "Please visit https://certbot.eff.org/ to check for other alternatives." "$VENV_BIN/letsencrypt" "$@" exit 0 @@ -1497,18 +1488,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.9.0 \ - --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \ - --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57 -acme==1.9.0 \ - --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \ - --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f -certbot-apache==1.9.0 \ - --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \ - --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e -certbot-nginx==1.9.0 \ - --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \ - --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65 +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-ci/certbot_integration_tests/assets/bind-config/conf/named.conf b/certbot-ci/certbot_integration_tests/assets/bind-config/conf/named.conf new file mode 100644 index 000000000..672a447d3 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/bind-config/conf/named.conf @@ -0,0 +1,60 @@ +options { + directory "/var/cache/bind"; + + // Running inside Docker. Bind address on Docker host is 127.0.0.1. + listen-on { any; }; + listen-on-v6 { any; }; + + // We are allowing BIND to service recursive queries, but only in an extremely limimited sense + // where it is entirely disconnected from public DNS: + // - Iterative queries are disabled. Only forwarding to a non-existent forwarder. + // - The only recursive answers we can get (that will not be a SERVFAIL) will come from the + // RPZ "mock-recursion" zone. Effectively this means we are mocking out the entirety of + // public DNS. + allow-recursion { any; }; // BIND will only answer using RPZ if recursion is enabled + forwarders { 192.0.2.254; }; // Nobody is listening, this is TEST-NET-1 + forward only; // Do NOT perform iterative queries from the root zone + dnssec-validation no; // Do not bother fetching the root DNSKEY set (performance) + response-policy { // All recursive queries will be served from here. + zone "mock-recursion" + log yes; + } recursive-only no // Allow RPZs to affect authoritative zones too. + qname-wait-recurse no // No real recursion. + nsip-wait-recurse no; // No real recursion. + + allow-transfer { none; }; + allow-update { none; }; +}; + +key "default-key." { + algorithm hmac-sha512; + secret "91CgOwzihr0nAVEHKFXJPQCbuBBbBI19Ks5VAweUXgbF40NWTD83naeg3c5y2MPdEiFRXnRLJxL6M+AfHCGLNw=="; +}; + +zone "mock-recursion" { + type primary; + file "/var/lib/bind/rpz.mock-recursion"; + allow-query { + none; + }; +}; + +zone "example.com." { + type primary; + file "/var/lib/bind/db.example.com"; + journal "/var/cache/bind/db.example.com.jnl"; + + update-policy { + grant default-key zonesub TXT; + }; +}; + +zone "sub.example.com." { + type primary; + file "/var/lib/bind/db.sub.example.com"; + journal "/var/cache/bind/db.sub.example.com.jnl"; + + update-policy { + grant default-key zonesub TXT; + }; +}; diff --git a/certbot-ci/certbot_integration_tests/assets/bind-config/rfc2136-credentials-default.ini.tpl b/certbot-ci/certbot_integration_tests/assets/bind-config/rfc2136-credentials-default.ini.tpl new file mode 100644 index 000000000..8aa7cc3cb --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/bind-config/rfc2136-credentials-default.ini.tpl @@ -0,0 +1,10 @@ +# Target DNS server +dns_rfc2136_server = {server_address} +# Target DNS port +dns_rfc2136_port = {server_port} +# TSIG key name +dns_rfc2136_name = default-key. +# TSIG key secret +dns_rfc2136_secret = 91CgOwzihr0nAVEHKFXJPQCbuBBbBI19Ks5VAweUXgbF40NWTD83naeg3c5y2MPdEiFRXnRLJxL6M+AfHCGLNw== +# TSIG key algorithm +dns_rfc2136_algorithm = HMAC-SHA512 diff --git a/certbot-ci/certbot_integration_tests/assets/bind-config/zones/db.example.com b/certbot-ci/certbot_integration_tests/assets/bind-config/zones/db.example.com new file mode 100644 index 000000000..2573470b5 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/bind-config/zones/db.example.com @@ -0,0 +1,11 @@ +$ORIGIN example.com. +$TTL 3600 +example.com. IN SOA ns1.example.com. admin.example.com. ( 2020091025 7200 3600 1209600 3600 ) + +example.com. IN NS ns1 +example.com. IN NS ns2 + +ns1 IN A 192.0.2.2 +ns2 IN A 192.0.2.3 + +@ IN A 192.0.2.1 diff --git a/certbot-ci/certbot_integration_tests/assets/bind-config/zones/db.sub.example.com b/certbot-ci/certbot_integration_tests/assets/bind-config/zones/db.sub.example.com new file mode 100644 index 000000000..0379003b7 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/bind-config/zones/db.sub.example.com @@ -0,0 +1,9 @@ +$ORIGIN sub.example.com. +$TTL 3600 +sub.example.com. IN SOA ns1.example.com. admin.example.com. ( 2020091025 7200 3600 1209600 3600 ) + +sub.example.com. IN NS ns1 +sub.example.com. IN NS ns2 + +ns1 IN A 192.0.2.2 +ns2 IN A 192.0.2.3 diff --git a/certbot-ci/certbot_integration_tests/assets/bind-config/zones/rpz.mock-recursion b/certbot-ci/certbot_integration_tests/assets/bind-config/zones/rpz.mock-recursion new file mode 100644 index 000000000..589689d37 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/bind-config/zones/rpz.mock-recursion @@ -0,0 +1,6 @@ +$TTL 3600 + +@ SOA ns1.example.test. dummy.example.test. 1 12h 15m 3w 2h + NS ns1.example.test. + +_acme-challenge.aliased.example IN CNAME _acme-challenge.example.com. diff --git a/certbot-ci/certbot_integration_tests/assets/hook.py b/certbot-ci/certbot_integration_tests/assets/hook.py index 483892a93..c11704f47 100755 --- a/certbot-ci/certbot_integration_tests/assets/hook.py +++ b/certbot-ci/certbot_integration_tests/assets/hook.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function import os import sys diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/README b/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/README new file mode 100644 index 000000000..5050078ff --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/README @@ -0,0 +1,14 @@ +This directory contains your keys and certificates. + +`privkey.pem` : the private key for your certificate. +`fullchain.pem`: the certificate file used in most server software. +`chain.pem` : used for OCSP stapling in Nginx >=1.3.7. +`cert.pem` : will break many server configurations, and should not be used + without reading further documentation (see link below). + +WARNING: DO NOT MOVE OR RENAME THESE FILES! + Certbot expects these files to remain in this location in order + to function properly! + +We recommend not moving these files. For more information, see the Certbot +User Guide at https://certbot.eff.org/docs/using.html#where-are-my-certificates. diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/cert.pem b/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/cert.pem new file mode 100644 index 000000000..b07af2bf7 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/cert.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC2zCCAcOgAwIBAgIIBvrEnbPRYu8wDQYJKoZIhvcNAQELBQAwKDEmMCQGA1UE +AxMdUGViYmxlIEludGVybWVkaWF0ZSBDQSAxMjZjNGIwHhcNMjAxMDEyMjEwNzQw +WhcNMjUxMDEyMjEwNzQwWjAjMSEwHwYDVQQDExhjLmVuY3J5cHRpb24tZXhhbXBs +ZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARjMhuW0ENPPC33PjB5XsYU +CRw640kPQENIDatcTJaENZIZdqKd6rI6jc+lpbmXot7Zi52clJlSJS+V6oDAt2Lh +o4HYMIHVMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB +BQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUj7Kd3ENqxlPf8B2bIGhsjydX +mPswHwYDVR0jBBgwFoAUEiGxlkRsi+VvcogH5dVD3h1laAcwMQYIKwYBBQUHAQEE +JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vMTI3LjAuMC4xOjQwMDIwIwYDVR0RBBww +GoIYYy5lbmNyeXB0aW9uLWV4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQCl +k0JXsa8y7fg41WWMDhw60bPW77O0FtOmTcnhdI5daYNemQVk+Q5EMaBLQ/oGjgXd +9QXFzXH1PL904YEnSLt+iTpXn++7rQSNzQsdYqw0neWk4f5pEBiN+WORpb6mwobV +ifMtBOkNEHvrJ2Pkci9U1lLwtKD/DSew6QtJU5DSkmH1XdGuMJiubygEIvELtvgq +cP9S368ZvPmPGmKaJQXBiuaR8MTjY/Bkr79aXQMjKbf+mpn7h0POCcePk1DY/rm6 +Da+X16lf0hHyQhSUa7Vgyim6rK1/hlw+Z00i+sQCKD9Ih7kXuuGqfSDC33cfO8Tj +o/MXO8lcxkrem5zU5QWP +-----END CERTIFICATE----- diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/chain.pem b/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/chain.pem new file mode 100644 index 000000000..64a805c0f --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/chain.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDUDCCAjigAwIBAgIIbi787yVrcMAwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE +AxMVUGViYmxlIFJvb3QgQ0EgMGM1MjI1MCAXDTIwMTAxMjIwMjI0NloYDzIwNTAx +MDEyMjEyMjQ2WjAoMSYwJAYDVQQDEx1QZWJibGUgSW50ZXJtZWRpYXRlIENBIDEy +NmM0YjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALGeVk1BMJraeqRq +mJ2+hgso8VOAv2s2CVxUJjIVcn7f2adE8NyTsSQ1brlsnKCUYUw7yLTQH0izLQRB +qKVIDFkUqo5/FuTJ2QlfA2EwBL8J7s/7L7vj3L0DiVpwgxPSyFEwdl/Y5y7ofsX5 +CIhCFcaMAmTIuKLiSfCJjGwkbEMuolm+lO8Mikxxc/JtDVUC479ugU7PU9O09bMH +nm+sD6Bgd+KMoPkCCCoeShJS9X3Ziq9HGc7Z6nhM/zirFARt2XkonEdAZ8br01zY +MRiY9txhlWQ7mUkOtzOSoEuYJNoUbvMUf0+tNzto26WRyF7dJmh7lTBsYrvAwUTx +PzNyst0CAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYB +BQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBIhsZZE +bIvlb3KIB+XVQ94dZWgHMB8GA1UdIwQYMBaAFOaKTaXg37vKgRt7d79YOjAoAtJT +MA0GCSqGSIb3DQEBCwUAA4IBAQAU2mZii7PH2pkw2lNM0QqPbcW/UYyvFoUeM8Aq +uCtsI2s+oxCJTqzfLsA0N8NY4nHLQ5wAlNJfJekngni8hbmJTKU4JFTMe7kLQO8P +fJbk0pTzhhHVQw7CVwB6Pwq3u2m/JV+d6xDIDc+AVkuEl19ZJU0rTWyooClfFLZV +EdZmEiUtA3PGlxoYwYhoGHYlhFxsoFONhCsBEdN7k7FKtFGVxN7oc5SKmKp0YZTW +fcrEtrdNThATO4ymhCC2zh33NI/MT1O74fpaAc2k6LcTl57MKiLfTYX4LTL6v9JG +9tlNqjFVRRmzEbtXTPcCb+w9g1VqoOGok7mGXYLTYtShCuvE +-----END CERTIFICATE----- diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/fullchain.pem b/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/fullchain.pem new file mode 100644 index 000000000..1ba80ba4e --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/fullchain.pem @@ -0,0 +1,38 @@ +-----BEGIN CERTIFICATE----- +MIIC2zCCAcOgAwIBAgIILlmGtZhUFEwwDQYJKoZIhvcNAQELBQAwKDEmMCQGA1UE +AxMdUGViYmxlIEludGVybWVkaWF0ZSBDQSAxMjZjNGIwHhcNMjAxMDEyMjA1MDM0 +WhcNMjUxMDEyMjA1MDM0WjAjMSEwHwYDVQQDExhjLmVuY3J5cHRpb24tZXhhbXBs +ZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARHEzR8JPWrEmpmgM+F2bk5 +9mT0u6CjzmJG0QpbaqprLiG5NGpW84VQ5TFCrmC4KxYfigCfMhfHRNfFYvNUK3V/ +o4HYMIHVMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB +BQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU1CsVL+bPnzaxxQ5jUENmQJIO +lKwwHwYDVR0jBBgwFoAUEiGxlkRsi+VvcogH5dVD3h1laAcwMQYIKwYBBQUHAQEE +JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vMTI3LjAuMC4xOjQwMDIwIwYDVR0RBBww +GoIYYy5lbmNyeXB0aW9uLWV4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQBn +2D8loC7pfk28JYpFLr5lmFKJWWmtLGlpsWDj61fVjtTfGKLziJz+MM6il4Y3hIz5 +58qiFK0ue0M63dIBJ33N+XxSEXon4Q0gy/zRWfH9jtPJ3FwfjkU/RT9PAUClYi0G +ptNWnTmgQkNzousbcAtRNXuuShH3856vhUnwkX+xM+cbIDi1JVmFjcGrEEQJ0rUF +mv2ZTyfbWbUs3v4rReETi2NVzr1Ql6J+ByNcMvHODzFy3t0L6yelAw2ca1I+c9HU ++Z0tnp/ykR7eXNuVLivok8UBf5OC413lh8ZO5g+Bgzh/LdtkUuavg1MYtEX0H6mX +9U7y3nVI8WEbPGf+HDeu +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDUDCCAjigAwIBAgIIbi787yVrcMAwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE +AxMVUGViYmxlIFJvb3QgQ0EgMGM1MjI1MCAXDTIwMTAxMjIwMjI0NloYDzIwNTAx +MDEyMjEyMjQ2WjAoMSYwJAYDVQQDEx1QZWJibGUgSW50ZXJtZWRpYXRlIENBIDEy +NmM0YjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALGeVk1BMJraeqRq +mJ2+hgso8VOAv2s2CVxUJjIVcn7f2adE8NyTsSQ1brlsnKCUYUw7yLTQH0izLQRB +qKVIDFkUqo5/FuTJ2QlfA2EwBL8J7s/7L7vj3L0DiVpwgxPSyFEwdl/Y5y7ofsX5 +CIhCFcaMAmTIuKLiSfCJjGwkbEMuolm+lO8Mikxxc/JtDVUC479ugU7PU9O09bMH +nm+sD6Bgd+KMoPkCCCoeShJS9X3Ziq9HGc7Z6nhM/zirFARt2XkonEdAZ8br01zY +MRiY9txhlWQ7mUkOtzOSoEuYJNoUbvMUf0+tNzto26WRyF7dJmh7lTBsYrvAwUTx +PzNyst0CAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYB +BQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBIhsZZE +bIvlb3KIB+XVQ94dZWgHMB8GA1UdIwQYMBaAFOaKTaXg37vKgRt7d79YOjAoAtJT +MA0GCSqGSIb3DQEBCwUAA4IBAQAU2mZii7PH2pkw2lNM0QqPbcW/UYyvFoUeM8Aq +uCtsI2s+oxCJTqzfLsA0N8NY4nHLQ5wAlNJfJekngni8hbmJTKU4JFTMe7kLQO8P +fJbk0pTzhhHVQw7CVwB6Pwq3u2m/JV+d6xDIDc+AVkuEl19ZJU0rTWyooClfFLZV +EdZmEiUtA3PGlxoYwYhoGHYlhFxsoFONhCsBEdN7k7FKtFGVxN7oc5SKmKp0YZTW +fcrEtrdNThATO4ymhCC2zh33NI/MT1O74fpaAc2k6LcTl57MKiLfTYX4LTL6v9JG +9tlNqjFVRRmzEbtXTPcCb+w9g1VqoOGok7mGXYLTYtShCuvE +-----END CERTIFICATE----- diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/privkey.pem b/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/privkey.pem new file mode 100644 index 000000000..6843b83d6 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/privkey.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgNgefv2dad4U1VYEi +0WkdHuqywi5QXAe30OwNTTGjhbihRANCAARHEzR8JPWrEmpmgM+F2bk59mT0u6Cj +zmJG0QpbaqprLiG5NGpW84VQ5TFCrmC4KxYfigCfMhfHRNfFYvNUK3V/ +-----END PRIVATE KEY----- diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/README b/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/README new file mode 100644 index 000000000..5050078ff --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/README @@ -0,0 +1,14 @@ +This directory contains your keys and certificates. + +`privkey.pem` : the private key for your certificate. +`fullchain.pem`: the certificate file used in most server software. +`chain.pem` : used for OCSP stapling in Nginx >=1.3.7. +`cert.pem` : will break many server configurations, and should not be used + without reading further documentation (see link below). + +WARNING: DO NOT MOVE OR RENAME THESE FILES! + Certbot expects these files to remain in this location in order + to function properly! + +We recommend not moving these files. For more information, see the Certbot +User Guide at https://certbot.eff.org/docs/using.html#where-are-my-certificates. diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/cert.pem b/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/cert.pem new file mode 120000 index 000000000..23e9f36ef --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/cert.pem @@ -0,0 +1 @@ +../../archive/c.encryption-example.com/cert.pem \ No newline at end of file diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/chain.pem b/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/chain.pem new file mode 120000 index 000000000..3ce63c220 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/chain.pem @@ -0,0 +1 @@ +../../archive/c.encryption-example.com/chain.pem \ No newline at end of file diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/fullchain.pem b/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/fullchain.pem new file mode 120000 index 000000000..5f86022af --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/fullchain.pem @@ -0,0 +1 @@ +../../archive/c.encryption-example.com/fullchain.pem \ No newline at end of file diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/privkey.pem b/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/privkey.pem new file mode 120000 index 000000000..4a8866fdc --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/privkey.pem @@ -0,0 +1 @@ +../../archive/c.encryption-example.com/privkey.pem \ No newline at end of file diff --git a/certbot-ci/certbot_integration_tests/assets/sample-config/renewal/c.encryption-example.com.conf b/certbot-ci/certbot_integration_tests/assets/sample-config/renewal/c.encryption-example.com.conf new file mode 100644 index 000000000..0c65891e4 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/assets/sample-config/renewal/c.encryption-example.com.conf @@ -0,0 +1,17 @@ +# renew_before_expiry = 30 days +version = 1.10.0.dev0 +archive_dir = sample-config/archive/c.encryption-example.com +cert = sample-config/live/c.encryption-example.com/cert.pem +privkey = sample-config/live/c.encryption-example.com/privkey.pem +chain = sample-config/live/c.encryption-example.com/chain.pem +fullchain = sample-config/live/c.encryption-example.com/fullchain.pem + +# Options used in the renewal process +[renewalparams] +authenticator = apache +installer = apache +account = 48d6b9e8d767eccf7e4d877d6ffa81e3 +key_type = ecdsa +config_dir = sample-config-ec +elliptic_curve = secp256r1 +manual_public_ip_logging_ok = True diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py b/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py index 60c2fcdd8..819cb3e78 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py @@ -1,3 +1,4 @@ +# pylint: disable=missing-module-docstring import pytest # Custom assertions defined in the following package need to be registered to be properly diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/assertions.py b/certbot-ci/certbot_integration_tests/certbot_tests/assertions.py index 53df6b890..c223d524c 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/assertions.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/assertions.py @@ -2,6 +2,11 @@ import io import os +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKey +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey +from cryptography.hazmat.primitives.serialization import load_pem_private_key + try: import grp POSIX_MODE = True @@ -16,6 +21,33 @@ SYSTEM_SID = 'S-1-5-18' ADMINS_SID = 'S-1-5-32-544' +def assert_elliptic_key(key, curve): + """ + Asserts that the key at the given path is an EC key using the given curve. + :param key: path to key + :param curve: name of the expected elliptic curve + """ + with open(key, 'rb') as file: + privkey1 = file.read() + + key = load_pem_private_key(data=privkey1, password=None, backend=default_backend()) + + assert isinstance(key, EllipticCurvePrivateKey) + assert isinstance(key.curve, curve) + + +def assert_rsa_key(key): + """ + Asserts that the key at the given path is an RSA key. + :param key: path to key + """ + with open(key, 'rb') as file: + privkey1 = file.read() + + key = load_pem_private_key(data=privkey1, password=None, backend=default_backend()) + assert isinstance(key, RSAPrivateKey) + + def assert_hook_execution(probe_path, probe_content): """ Assert that a certbot hook has been executed diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index e295aefd7..b052e375d 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -7,7 +7,7 @@ import tempfile from certbot_integration_tests.utils import certbot_call -class IntegrationTestsContext(object): +class IntegrationTestsContext: """General fixture describing a certbot integration tests context""" def __init__(self, request): self.request = request @@ -77,6 +77,6 @@ class IntegrationTestsContext(object): appending the pytest worker id to the subdomain, using this pattern: {subdomain}.{worker_id}.wtf :param subdomain: the subdomain to use in the generated domain (default 'le') - :return: the well-formed domain suitable for redirection on + :return: the well-formed domain suitable for redirection on """ return '{0}.{1}.wtf'.format(subdomain, self.worker_id) diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index cfafce732..0b649acdc 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -1,5 +1,4 @@ """Module executing integration tests against certbot core.""" -from __future__ import print_function import os from os.path import exists @@ -9,12 +8,15 @@ import shutil import subprocess import time +from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1, SECP384R1, SECP521R1 from cryptography.x509 import NameOID import pytest from certbot_integration_tests.certbot_tests import context as certbot_context from certbot_integration_tests.certbot_tests.assertions import assert_cert_count_for_lineage +from certbot_integration_tests.certbot_tests.assertions import assert_elliptic_key +from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_owner from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_permissions from certbot_integration_tests.certbot_tests.assertions import assert_equals_world_read_permissions @@ -26,8 +28,9 @@ from certbot_integration_tests.certbot_tests.assertions import EVERYBODY_SID from certbot_integration_tests.utils import misc -@pytest.fixture() -def context(request): +@pytest.fixture(name='context') +def test_context(request): + # pylint: disable=missing-function-docstring # Fixture request is a built-in pytest fixture describing current test request. integration_test_context = certbot_context.IntegrationTestsContext(request) try: @@ -144,6 +147,17 @@ def test_certonly(context): """Test the certonly verb on certbot.""" context.certbot(['certonly', '--cert-name', 'newname', '-d', context.get_domain('newname')]) + assert_cert_count_for_lineage(context.config_dir, 'newname', 1) + + +def test_certonly_webroot(context): + """Test the certonly verb with webroot plugin""" + with misc.create_http_server(context.http_01_port) as webroot: + certname = context.get_domain('webroot') + context.certbot(['certonly', '-a', 'webroot', '--webroot-path', webroot, '-d', certname]) + + assert_cert_count_for_lineage(context.config_dir, certname, 1) + def test_auth_and_install_with_csr(context): """Test certificate issuance and install using an existing CSR.""" @@ -219,14 +233,16 @@ def test_renew_files_propagate_permissions(context): if os.name != 'nt': os.chmod(privkey1, 0o444) else: - import win32security - import ntsecuritycon + import win32security # pylint: disable=import-error + import ntsecuritycon # pylint: disable=import-error # Get the current DACL of the private key security = win32security.GetFileSecurity(privkey1, win32security.DACL_SECURITY_INFORMATION) dacl = security.GetSecurityDescriptorDacl() # Create a read permission for Everybody group everybody = win32security.ConvertStringSidToSid(EVERYBODY_SID) - dacl.AddAccessAllowedAce(win32security.ACL_REVISION, ntsecuritycon.FILE_GENERIC_READ, everybody) + dacl.AddAccessAllowedAce( + win32security.ACL_REVISION, ntsecuritycon.FILE_GENERIC_READ, everybody + ) # Apply the updated DACL to the private key security.SetSecurityDescriptorDacl(1, dacl, 0) win32security.SetFileSecurity(privkey1, win32security.DACL_SECURITY_INFORMATION, security) @@ -235,12 +251,14 @@ def test_renew_files_propagate_permissions(context): assert_cert_count_for_lineage(context.config_dir, certname, 2) if os.name != 'nt': - # On Linux, read world permissions + all group permissions will be copied from the previous private key + # On Linux, read world permissions + all group permissions + # will be copied from the previous private key assert_world_read_permissions(privkey2) assert_equals_world_read_permissions(privkey1, privkey2) assert_equals_group_permissions(privkey1, privkey2) else: - # On Windows, world will never have any permissions, and group permission is irrelevant for this platform + # On Windows, world will never have any permissions, and + # group permission is irrelevant for this platform assert_world_no_permissions(privkey2) @@ -289,7 +307,7 @@ def test_renew_with_changed_private_key_complexity(context): assert_cert_count_for_lineage(context.config_dir, certname, 1) context.certbot(['renew']) - + assert_cert_count_for_lineage(context.config_dir, certname, 2) key2 = join(context.config_dir, 'archive', certname, 'privkey2.pem') assert os.stat(key2).st_size > 3000 @@ -421,20 +439,115 @@ def test_reuse_key(context): assert len({cert1, cert2, cert3}) == 3 +def test_incorrect_key_type(context): + with pytest.raises(subprocess.CalledProcessError): + context.certbot(['--key-type="failwhale"']) + + def test_ecdsa(context): - """Test certificate issuance with ECDSA key.""" + """Test issuance for ECDSA CSR based request (legacy supported mode).""" key_path = join(context.workspace, 'privkey-p384.pem') csr_path = join(context.workspace, 'csr-p384.der') cert_path = join(context.workspace, 'cert-p384.pem') chain_path = join(context.workspace, 'chain-p384.pem') - misc.generate_csr([context.get_domain('ecdsa')], key_path, csr_path, key_type=misc.ECDSA_KEY_TYPE) - context.certbot(['auth', '--csr', csr_path, '--cert-path', cert_path, '--chain-path', chain_path]) + misc.generate_csr( + [context.get_domain('ecdsa')], + key_path, csr_path, + key_type=misc.ECDSA_KEY_TYPE + ) + context.certbot([ + 'auth', '--csr', csr_path, '--cert-path', cert_path, + '--chain-path', chain_path, + ]) certificate = misc.read_certificate(cert_path) assert 'ASN1 OID: secp384r1' in certificate +def test_default_key_type(context): + """Test default key type is RSA""" + certname = context.get_domain('renew') + context.certbot([ + 'certonly', + '--cert-name', certname, '-d', certname + ]) + filename = join(context.config_dir, 'archive/{0}/privkey1.pem').format(certname) + assert_rsa_key(filename) + + +def test_default_curve_type(context): + """test that the curve used when not specifying any is secp256r1""" + certname = context.get_domain('renew') + context.certbot([ + '--key-type', 'ecdsa', '--cert-name', certname, '-d', certname + ]) + key1 = join(context.config_dir, 'archive/{0}/privkey1.pem'.format(certname)) + assert_elliptic_key(key1, SECP256R1) + + +@pytest.mark.parametrize('curve,curve_cls,skip_servers', [ + # Curve name, Curve class, ACME servers to skip + ('secp256r1', SECP256R1, []), + ('secp384r1', SECP384R1, []), + ('secp521r1', SECP521R1, ['boulder-v1', 'boulder-v2'])] +) +def test_ecdsa_curves(context, curve, curve_cls, skip_servers): + """Test issuance for each supported ECDSA curve""" + if context.acme_server in skip_servers: + pytest.skip('ACME server {} does not support ECDSA curve {}' + .format(context.acme_server, curve)) + + domain = context.get_domain('curve') + context.certbot([ + 'certonly', + '--key-type', 'ecdsa', '--elliptic-curve', curve, + '--force-renewal', '-d', domain, + ]) + key = join(context.config_dir, "live", domain, 'privkey.pem') + assert_elliptic_key(key, curve_cls) + + +def test_renew_with_ec_keys(context): + """Test proper renew with updated private key complexity.""" + certname = context.get_domain('renew') + context.certbot([ + 'certonly', + '--cert-name', certname, + '--key-type', 'ecdsa', '--elliptic-curve', 'secp256r1', + '--force-renewal', '-d', certname, + ]) + + key1 = join(context.config_dir, "archive", certname, 'privkey1.pem') + assert 200 < os.stat(key1).st_size < 250 # ec keys of 256 bits are ~225 bytes + assert_elliptic_key(key1, SECP256R1) + assert_cert_count_for_lineage(context.config_dir, certname, 1) + + context.certbot(['renew', '--elliptic-curve', 'secp384r1']) + + assert_cert_count_for_lineage(context.config_dir, certname, 2) + key2 = join(context.config_dir, 'archive', certname, 'privkey2.pem') + assert_elliptic_key(key2, SECP384R1) + assert 280 < os.stat(key2).st_size < 320 # ec keys of 384 bits are ~310 bytes + + # We expect here that the command will fail because without --key-type specified, + # Certbot must error out to prevent changing an existing certificate key type, + # without explicit user consent (by specifying both --cert-name and --key-type). + with pytest.raises(subprocess.CalledProcessError): + context.certbot([ + 'certonly', + '--force-renewal', + '-d', certname + ]) + + # We expect that the previous behavior of requiring both --cert-name and + # --key-type to be set to not apply to the renew subcommand. + context.certbot(['renew', '--force-renewal', '--key-type', 'rsa']) + assert_cert_count_for_lineage(context.config_dir, certname, 3) + key3 = join(context.config_dir, 'archive', certname, 'privkey3.pem') + assert_rsa_key(key3) + + def test_ocsp_must_staple(context): """Test that OCSP Must-Staple is correctly set in the generated certificate.""" if context.acme_server == 'pebble': @@ -533,19 +646,22 @@ def test_revoke_multiple_lineages(context): with open(join(context.config_dir, 'renewal', '{0}.conf'.format(cert2)), 'r') as file: data = file.read() - data = re.sub('archive_dir = .*\n', - 'archive_dir = {0}\n'.format(join(context.config_dir, 'archive', cert1).replace('\\', '\\\\')), - data) + data = re.sub( + 'archive_dir = .*\n', + 'archive_dir = {0}\n'.format( + join(context.config_dir, 'archive', cert1).replace('\\', '\\\\') + ), data + ) with open(join(context.config_dir, 'renewal', '{0}.conf'.format(cert2)), 'w') as file: file.write(data) - output = context.certbot([ + context.certbot([ 'revoke', '--cert-path', join(context.config_dir, 'live', cert1, 'cert.pem') ]) with open(join(context.workspace, 'logs', 'letsencrypt.log'), 'r') as f: - assert 'Not deleting revoked certs due to overlapping archive dirs' in f.read() + assert 'Not deleting revoked certificates due to overlapping archive dirs' in f.read() def test_wildcard_certificates(context): @@ -658,4 +774,4 @@ def test_preferred_chain(context): with open(conf_path, 'r') as f: assert 'preferred_chain = {}'.format(requested) in f.read(), \ - 'Expected preferred_chain to be set in renewal config' \ No newline at end of file + 'Expected preferred_chain to be set in renewal config' diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index bb1d76e57..6fc77480d 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -6,12 +6,12 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ -from __future__ import print_function import contextlib import subprocess import sys from certbot_integration_tests.utils import acme_server as acme_lib +from certbot_integration_tests.utils import dns_server as dns_lib def pytest_addoption(parser): @@ -23,6 +23,10 @@ def pytest_addoption(parser): choices=['boulder-v1', 'boulder-v2', 'pebble'], help='select the ACME server to use (boulder-v1, boulder-v2, ' 'pebble), defaulting to pebble') + parser.addoption('--dns-server', default='challtestsrv', + choices=['bind', 'challtestsrv'], + help='select the DNS server to use (bind, challtestsrv), ' + 'defaulting to challtestsrv') def pytest_configure(config): @@ -32,7 +36,7 @@ def pytest_configure(config): """ if not hasattr(config, 'slaveinput'): # If true, this is the primary node with _print_on_err(): - config.acme_xdist = _setup_primary_node(config) + _setup_primary_node(config) def pytest_configure_node(node): @@ -41,6 +45,7 @@ def pytest_configure_node(node): :param node: current worker node """ node.slaveinput['acme_xdist'] = node.config.acme_xdist + node.slaveinput['dns_xdist'] = node.config.dns_xdist @contextlib.contextmanager @@ -61,12 +66,18 @@ def _print_on_err(): def _setup_primary_node(config): """ Setup the environment for integration tests. - Will: + + This function will: - check runtime compatibility (Docker, docker-compose, Nginx) - create a temporary workspace and the persistent GIT repositories space + - configure and start a DNS server using Docker, if configured - configure and start paralleled ACME CA servers using Docker - - transfer ACME CA servers configurations to pytest nodes using env variables - :param config: Configuration of the pytest primary node + - transfer ACME CA and DNS servers configurations to pytest nodes using env variables + + This function modifies `config` by injecting the ACME CA and DNS server configurations, + in addition to cleanup functions for those servers. + + :param config: Configuration of the pytest primary node. Is modified by this function. """ # Check for runtime compatibility: some tools are required to be available in PATH if 'boulder' in config.option.acme_server: @@ -79,18 +90,35 @@ def _setup_primary_node(config): try: subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT) except (subprocess.CalledProcessError, OSError): - raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, ' - 'but is not installed or not available for current user.') + raise ValueError( + 'Error: docker-compose is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.' + ) # Parameter numprocesses is added to option by pytest-xdist workers = ['primary'] if not config.option.numprocesses\ else ['gw{0}'.format(i) for i in range(config.option.numprocesses)] + # If a non-default DNS server is configured, start it and feed it to the ACME server + dns_server = None + acme_dns_server = None + if config.option.dns_server == 'bind': + dns_server = dns_lib.DNSServer(workers) + config.add_cleanup(dns_server.stop) + print('DNS xdist config:\n{0}'.format(dns_server.dns_xdist)) + dns_server.start() + acme_dns_server = '{}:{}'.format( + dns_server.dns_xdist['address'], + dns_server.dns_xdist['port'] + ) + # By calling setup_acme_server we ensure that all necessary acme server instances will be # fully started. This runtime is reflected by the acme_xdist returned. - acme_server = acme_lib.ACMEServer(config.option.acme_server, workers) + acme_server = acme_lib.ACMEServer(config.option.acme_server, workers, + dns_server=acme_dns_server) config.add_cleanup(acme_server.stop) print('ACME xdist config:\n{0}'.format(acme_server.acme_xdist)) acme_server.start() - return acme_server.acme_xdist + config.acme_xdist = acme_server.acme_xdist + config.dns_xdist = dns_server.dns_xdist if dns_server else None diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/context.py b/certbot-ci/certbot_integration_tests/nginx_tests/context.py index 3a769840c..6f0f833a0 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/context.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/context.py @@ -1,3 +1,4 @@ +"""Module to handle the context of nginx integration tests.""" import os import subprocess diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py b/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py index 18991ae62..bbbe8ea06 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """General purpose nginx test configuration generator.""" import getpass @@ -42,6 +43,8 @@ events {{ worker_connections 1024; }} +# “This comment contains valid Unicode”. + http {{ # Set an array of temp, cache and log file options that will otherwise default to # restricted locations accessible only to root. @@ -51,61 +54,61 @@ http {{ #scgi_temp_path {nginx_root}/scgi_temp; #uwsgi_temp_path {nginx_root}/uwsgi_temp; access_log {nginx_root}/error.log; - + # This should be turned off in a Virtualbox VM, as it can cause some # interesting issues with data corruption in delivered files. sendfile off; - + tcp_nopush on; tcp_nodelay on; keepalive_timeout 65; types_hash_max_size 2048; - + #include /etc/nginx/mime.types; index index.html index.htm index.php; - + log_format main '$remote_addr - $remote_user [$time_local] $status ' '"$request" $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; - + default_type application/octet-stream; - + server {{ # IPv4. listen {http_port} {default_server}; # IPv6. listen [::]:{http_port} {default_server}; server_name nginx.{wtf_prefix}.wtf nginx2.{wtf_prefix}.wtf; - + root {nginx_webroot}; - + location / {{ # First attempt to serve request as file, then as directory, then fall # back to index.html. try_files $uri $uri/ /index.html; }} }} - + server {{ listen {http_port}; listen [::]:{http_port}; server_name nginx3.{wtf_prefix}.wtf; - + root {nginx_webroot}; - + location /.well-known/ {{ return 404; }} - + return 301 https://$host$request_uri; }} - + server {{ listen {other_port}; listen [::]:{other_port}; server_name nginx4.{wtf_prefix}.wtf nginx5.{wtf_prefix}.wtf; }} - + server {{ listen {http_port}; listen [::]:{http_port}; diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py index 1a62ea8d7..8a2d48a50 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -2,13 +2,14 @@ import os import ssl +from typing import List import pytest from certbot_integration_tests.nginx_tests import context as nginx_context -@pytest.fixture() -def context(request): +@pytest.fixture(name='context') +def test_context(request): # Fixture request is a built-in pytest fixture describing current test request. integration_test_context = nginx_context.IntegrationTestsContext(request) try: @@ -27,10 +28,12 @@ def context(request): # No matching server block; default_server does not exist ('nginx5.{0}.wtf', ['--preferred-challenges', 'http'], {'default_server': False}), # Multiple domains, mix of matching and not - ('nginx6.{0}.wtf,nginx7.{0}.wtf', ['--preferred-challenges', 'http'], {'default_server': False}), + ('nginx6.{0}.wtf,nginx7.{0}.wtf', [ + '--preferred-challenges', 'http' + ], {'default_server': False}), ], indirect=['context']) def test_certificate_deployment(certname_pattern, params, context): - # type: (str, list, nginx_context.IntegrationTestsContext) -> None + # type: (str, List[str], nginx_context.IntegrationTestsContext) -> None """ Test various scenarios to deploy a certificate to nginx using certbot. """ @@ -41,7 +44,9 @@ def test_certificate_deployment(certname_pattern, params, context): lineage = domains.split(',')[0] server_cert = ssl.get_server_certificate(('localhost', context.tls_alpn_01_port)) - with open(os.path.join(context.workspace, 'conf/live/{0}/cert.pem'.format(lineage)), 'r') as file: + with open(os.path.join( + context.workspace, 'conf/live/{0}/cert.pem'.format(lineage)), 'r' + ) as file: certbot_cert = file.read() assert server_cert == certbot_cert diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/__init__.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py new file mode 100644 index 000000000..bdedee1fe --- /dev/null +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -0,0 +1,66 @@ +"""Module to handle the context of RFC2136 integration tests.""" + +import tempfile +from contextlib import contextmanager + +from pkg_resources import resource_filename +from pytest import skip + +from certbot_integration_tests.certbot_tests import context as certbot_context +from certbot_integration_tests.utils import certbot_call + + +class IntegrationTestsContext(certbot_context.IntegrationTestsContext): + """Integration test context for certbot-dns-rfc2136""" + def __init__(self, request): + super(IntegrationTestsContext, self).__init__(request) + + self.request = request + + self._dns_xdist = None + if hasattr(request.config, 'slaveinput'): # Worker node + self._dns_xdist = request.config.slaveinput['dns_xdist'] + else: # Primary node + self._dns_xdist = request.config.dns_xdist + + def certbot_test_rfc2136(self, args): + """ + Main command to execute certbot using the RFC2136 DNS authenticator. + :param list args: list of arguments to pass to Certbot + """ + command = ['--authenticator', 'dns-rfc2136', '--dns-rfc2136-propagation-seconds', '2'] + command.extend(args) + return certbot_call.certbot_test( + command, self.directory_url, self.http_01_port, self.tls_alpn_01_port, + self.config_dir, self.workspace, force_renew=True) + + @contextmanager + def rfc2136_credentials(self, label='default'): + """ + Produces the contents of a certbot-dns-rfc2136 credentials file. + :param str label: which RFC2136 credential to use + :yields: Path to credentials file + :rtype: str + """ + src_file = resource_filename('certbot_integration_tests', + 'assets/bind-config/rfc2136-credentials-{}.ini.tpl' + .format(label)) + contents = None + + with open(src_file, 'r') as f: + contents = f.read().format( + server_address=self._dns_xdist['address'], + server_port=self._dns_xdist['port'] + ) + + with tempfile.NamedTemporaryFile('w+', prefix='rfc2136-creds-{}'.format(label), + suffix='.ini', dir=self.workspace) as fp: + fp.write(contents) + fp.flush() + yield fp.name + + def skip_if_no_bind9_server(self): + """Skips the test if there was no RFC2136-capable DNS server configured + in the test environment""" + if not self._dns_xdist: + skip('No RFC2136-capable DNS server is configured') diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/test_main.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/test_main.py new file mode 100644 index 000000000..ae6c0018e --- /dev/null +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/test_main.py @@ -0,0 +1,26 @@ +"""Module executing integration tests against Certbot with the RFC2136 DNS authenticator.""" +import pytest + +from certbot_integration_tests.rfc2136_tests import context as rfc2136_context + + +@pytest.fixture(name="context") +def pytest_context(request): + # pylint: disable=missing-function-docstring + # Fixture request is a built-in pytest fixture describing current test request. + integration_test_context = rfc2136_context.IntegrationTestsContext(request) + try: + yield integration_test_context + finally: + integration_test_context.cleanup() + + +@pytest.mark.parametrize('domain', [('example.com'), ('sub.example.com')]) +def test_get_certificate(domain, context): + context.skip_if_no_bind9_server() + + with context.rfc2136_credentials() as creds: + context.certbot_test_rfc2136([ + 'certonly', '--dns-rfc2136-credentials', creds, + '-d', domain, '-d', '*.{}'.format(domain) + ]) diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index c73b6c895..b16aa80af 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """Module to setup an ACME CA server environment able to run multiple tests in parallel""" -from __future__ import print_function +import argparse import errno import json import os @@ -12,15 +12,17 @@ import sys import tempfile import time +from typing import List import requests from certbot_integration_tests.utils import misc from certbot_integration_tests.utils import pebble_artifacts from certbot_integration_tests.utils import proxy +# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils.constants import * -class ACMEServer(object): +class ACMEServer: """ ACMEServer configures and handles the lifecycle of an ACME CA server and an HTTP reverse proxy instance, to allow parallel execution of integration tests against the unique http-01 port @@ -29,24 +31,34 @@ class ACMEServer(object): ACMEServer gives access the acme_xdist parameter, listing the ports and directory url to use for each pytest node. It exposes also start and stop methods in order to start the stack, and stop it with proper resources cleanup. - ACMEServer is also a context manager, and so can be used to ensure ACME server is started/stopped - upon context enter/exit. + ACMEServer is also a context manager, and so can be used to ensure ACME server is + started/stopped upon context enter/exit. """ - def __init__(self, acme_server, nodes, http_proxy=True, stdout=False): + def __init__(self, acme_server, nodes, http_proxy=True, stdout=False, + dns_server=None, http_01_port=DEFAULT_HTTP_01_PORT): """ Create an ACMEServer instance. :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) :param list nodes: list of node names that will be setup by pytest xdist :param bool http_proxy: if False do not start the HTTP proxy :param bool stdout: if True stream all subprocesses stdout to standard stdout + :param str dns_server: if set, Pebble/Boulder will use it to resolve domains + :param int http_01_port: port to use for http-01 validation; currently + only supported for pebble without an HTTP proxy """ self._construct_acme_xdist(acme_server, nodes) self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' self._proxy = http_proxy self._workspace = tempfile.mkdtemp() - self._processes = [] + self._processes = [] # type: List[subprocess.Popen] self._stdout = sys.stdout if stdout else open(os.devnull, 'w') + self._dns_server = dns_server + self._http_01_port = http_01_port + if http_01_port != DEFAULT_HTTP_01_PORT: + if self._acme_type != 'pebble' or self._proxy: + raise ValueError('setting http_01_port is not currently supported ' + 'with boulder or the HTTP proxy') def start(self): """Start the test stack""" @@ -103,26 +115,34 @@ class ACMEServer(object): """Generate and return the acme_xdist dict""" acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT} - # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble. + # Directory and ACME port are set implicitly in the docker-compose.yml + # files of Boulder/Pebble. if acme_server == 'pebble': acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL else: # boulder acme_xdist['directory_url'] = BOULDER_V2_DIRECTORY_URL \ if acme_server == 'boulder-v2' else BOULDER_V1_DIRECTORY_URL - acme_xdist['http_port'] = {node: port for (node, port) - in zip(nodes, range(5200, 5200 + len(nodes)))} - acme_xdist['https_port'] = {node: port for (node, port) - in zip(nodes, range(5100, 5100 + len(nodes)))} - acme_xdist['other_port'] = {node: port for (node, port) - in zip(nodes, range(5300, 5300 + len(nodes)))} + acme_xdist['http_port'] = { + node: port for (node, port) in # pylint: disable=unnecessary-comprehension + zip(nodes, range(5200, 5200 + len(nodes))) + } + acme_xdist['https_port'] = { + node: port for (node, port) in # pylint: disable=unnecessary-comprehension + zip(nodes, range(5100, 5100 + len(nodes))) + } + acme_xdist['other_port'] = { + node: port for (node, port) in # pylint: disable=unnecessary-comprehension + zip(nodes, range(5300, 5300 + len(nodes))) + } self.acme_xdist = acme_xdist def _prepare_pebble_server(self): """Configure and launch the Pebble server""" print('=> Starting pebble instance deployment...') - pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts.fetch(self._workspace) + pebble_artifacts_rv = pebble_artifacts.fetch(self._workspace, self._http_01_port) + pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts_rv # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment. @@ -132,18 +152,23 @@ class ACMEServer(object): environ['PEBBLE_AUTHZREUSE'] = '100' environ['PEBBLE_ALTERNATE_ROOTS'] = str(PEBBLE_ALTERNATE_ROOTS) + if self._dns_server: + dns_server = self._dns_server + else: + dns_server = '127.0.0.1:8053' + self._launch_process( + [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT), + '-defaultIPv6', '""', '-defaultIPv4', '127.0.0.1', '-http01', '""', + '-tlsalpn01', '""', '-https01', '""']) + self._launch_process( - [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053', '-strict'], + [pebble_path, '-config', pebble_config_path, '-dnsserver', dns_server, '-strict'], env=environ) - self._launch_process( - [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT), '-defaultIPv6', '""', - '-defaultIPv4', '127.0.0.1', '-http01', '""', '-tlsalpn01', '""', '-https01', '""']) - - # pebble_ocsp_server is imported here and not at the top of module in order to avoid a useless - # ImportError, in the case where cryptography dependency is too old to support ocsp, but - # Boulder is used instead of Pebble, so pebble_ocsp_server is not used. This is the typical - # situation of integration-certbot-oldest tox testenv. + # pebble_ocsp_server is imported here and not at the top of module in order to avoid a + # useless ImportError, in the case where cryptography dependency is too old to support + # ocsp, but Boulder is used instead of Pebble, so pebble_ocsp_server is not used. This is + # the typical situation of integration-certbot-oldest tox testenv. from certbot_integration_tests.utils import pebble_ocsp_server self._launch_process([sys.executable, pebble_ocsp_server.__file__]) @@ -167,6 +192,15 @@ class ACMEServer(object): os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'), join(instance_path, 'test/rate-limit-policies.yml')) + if self._dns_server: + # Change Boulder config to use the provided DNS server + for suffix in ["", "-remote-a", "-remote-b"]: + with open(join(instance_path, 'test/config/va{}.json'.format(suffix)), 'r') as f: + config = json.loads(f.read()) + config['va']['dnsResolvers'] = [self._dns_server] + with open(join(instance_path, 'test/config/va{}.json'.format(suffix)), 'w') as f: + f.write(json.dumps(config, indent=2, separators=(',', ': '))) + try: # Launch the Boulder server self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path) @@ -175,14 +209,18 @@ class ACMEServer(object): print('=> Waiting for boulder instance to respond...') misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300) - # Configure challtestsrv to answer any A record request with ip of the docker host. - response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), - json={'ip': '10.77.77.1'}) - response.raise_for_status() + if not self._dns_server: + # Configure challtestsrv to answer any A record request with ip of the docker host. + response = requests.post('http://localhost:{0}/set-default-ipv4'.format( + CHALLTESTSRV_PORT), json={'ip': '10.77.77.1'} + ) + response.raise_for_status() except BaseException: # If we failed to set up boulder, print its logs. print('=> Boulder setup failed. Boulder logs are:') - process = self._launch_process(['docker-compose', 'logs'], cwd=instance_path, force_stderr=True) + process = self._launch_process([ + 'docker-compose', 'logs'], cwd=instance_path, force_stderr=True + ) process.wait() raise @@ -193,7 +231,7 @@ class ACMEServer(object): print('=> Configuring the HTTP proxy...') mapping = {r'.+\.{0}\.wtf'.format(node): 'http://127.0.0.1:{0}'.format(port) for node, port in self.acme_xdist['http_port'].items()} - command = [sys.executable, proxy.__file__, str(HTTP_01_PORT), json.dumps(mapping)] + command = [sys.executable, proxy.__file__, str(DEFAULT_HTTP_01_PORT), json.dumps(mapping)] self._launch_process(command) print('=> Finished configuring the HTTP proxy.') @@ -202,20 +240,34 @@ class ACMEServer(object): if not env: env = os.environ stdout = sys.stderr if force_stderr else self._stdout - process = subprocess.Popen(command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env) + process = subprocess.Popen( + command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env + ) self._processes.append(process) return process def main(): - args = sys.argv[1:] - server_type = args[0] if args else 'pebble' - possible_values = ('pebble', 'boulder-v1', 'boulder-v2') - if server_type not in possible_values: - raise ValueError('Invalid server value {0}, should be one of {1}' - .format(server_type, possible_values)) + # pylint: disable=missing-function-docstring + parser = argparse.ArgumentParser( + description='CLI tool to start a local instance of Pebble or Boulder CA server.') + parser.add_argument('--server-type', '-s', + choices=['pebble', 'boulder-v1', 'boulder-v2'], default='pebble', + help='type of CA server to start: can be Pebble or Boulder ' + '(in ACMEv1 or ACMEv2 mode), Pebble is used if not set.') + parser.add_argument('--dns-server', '-d', + help='specify the DNS server as `IP:PORT` to use by ' + 'Pebble; if not specified, a local mock DNS server will be used to ' + 'resolve domains to localhost.') + parser.add_argument('--http-01-port', type=int, default=DEFAULT_HTTP_01_PORT, + help='specify the port to use for http-01 validation; ' + 'this is currently only supported for Pebble.') + args = parser.parse_args() - acme_server = ACMEServer(server_type, [], http_proxy=False, stdout=True) + acme_server = ACMEServer( + args.server_type, [], http_proxy=False, stdout=True, + dns_server=args.dns_server, http_01_port=args.http_01_port, + ) try: with acme_server as acme_xdist: diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py index a71c610e5..b319eca4c 100755 --- a/certbot-ci/certbot_integration_tests/utils/certbot_call.py +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -1,13 +1,13 @@ #!/usr/bin/env python """Module to call certbot in test mode""" -from __future__ import absolute_import -from distutils.version import LooseVersion import os import subprocess import sys +from distutils.version import LooseVersion import certbot_integration_tests +# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils.constants import * @@ -35,6 +35,8 @@ def certbot_test(certbot_args, directory_url, http_01_port, tls_alpn_01_port, def _prepare_environ(workspace): + # pylint: disable=missing-function-docstring + new_environ = os.environ.copy() new_environ['TMPDIR'] = workspace @@ -58,8 +60,13 @@ def _prepare_environ(workspace): # certbot_integration_tests.__file__ is: # '/path/to/certbot/certbot-ci/certbot_integration_tests/__init__.pyc' # ... and we want '/path/to/certbot' - certbot_root = os.path.dirname(os.path.dirname(os.path.dirname(certbot_integration_tests.__file__))) - python_paths = [path for path in new_environ['PYTHONPATH'].split(':') if path != certbot_root] + certbot_root = os.path.dirname(os.path.dirname( + os.path.dirname(certbot_integration_tests.__file__)) + ) + python_paths = [ + path for path in new_environ['PYTHONPATH'].split(':') + if path != certbot_root + ] new_environ['PYTHONPATH'] = ':'.join(python_paths) return new_environ @@ -70,7 +77,8 @@ def _compute_additional_args(workspace, environ, force_renew): output = subprocess.check_output(['certbot', '--version'], universal_newlines=True, stderr=subprocess.STDOUT, cwd=workspace, env=environ) - version_str = output.split(' ')[1].strip() # Typical response is: output = 'certbot 0.31.0.dev0' + # Typical response is: output = 'certbot 0.31.0.dev0' + version_str = output.split(' ')[1].strip() if LooseVersion(version_str) >= LooseVersion('0.30.0'): additional_args.append('--no-random-sleep-on-renew') @@ -92,6 +100,7 @@ def _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_por '--no-verify-ssl', '--http-01-port', str(http_01_port), '--https-port', str(tls_alpn_01_port), + '--manual-public-ip-logging-ok', '--config-dir', config_dir, '--work-dir', os.path.join(workspace, 'work'), '--logs-dir', os.path.join(workspace, 'logs'), @@ -112,11 +121,12 @@ def _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_por def main(): + # pylint: disable=missing-function-docstring args = sys.argv[1:] # Default config is pebble directory_url = os.environ.get('SERVER', PEBBLE_DIRECTORY_URL) - http_01_port = int(os.environ.get('HTTP_01_PORT', HTTP_01_PORT)) + http_01_port = int(os.environ.get('HTTP_01_PORT', DEFAULT_HTTP_01_PORT)) tls_alpn_01_port = int(os.environ.get('TLS_ALPN_01_PORT', TLS_ALPN_01_PORT)) # Execution of certbot in a self-contained workspace diff --git a/certbot-ci/certbot_integration_tests/utils/constants.py b/certbot-ci/certbot_integration_tests/utils/constants.py index 8b002478e..b02c434db 100644 --- a/certbot-ci/certbot_integration_tests/utils/constants.py +++ b/certbot-ci/certbot_integration_tests/utils/constants.py @@ -1,5 +1,5 @@ """Some useful constants to use throughout certbot-ci integration tests""" -HTTP_01_PORT = 5002 +DEFAULT_HTTP_01_PORT = 5002 TLS_ALPN_01_PORT = 5001 CHALLTESTSRV_PORT = 8055 BOULDER_V1_DIRECTORY_URL = 'http://localhost:4000/directory' @@ -7,4 +7,4 @@ BOULDER_V2_DIRECTORY_URL = 'http://localhost:4001/directory' PEBBLE_DIRECTORY_URL = 'https://localhost:14000/dir' PEBBLE_MANAGEMENT_URL = 'https://localhost:15000' MOCK_OCSP_SERVER_PORT = 4002 -PEBBLE_ALTERNATE_ROOTS = 2 \ No newline at end of file +PEBBLE_ALTERNATE_ROOTS = 2 diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py new file mode 100644 index 000000000..eaa601f1b --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +"""Module to setup an RFC2136-capable DNS server""" +import os +import os.path +import shutil +import socket +import subprocess +import sys +import tempfile +import time + +from pkg_resources import resource_filename + +BIND_DOCKER_IMAGE = "internetsystemsconsortium/bind9:9.16" +BIND_BIND_ADDRESS = ("127.0.0.1", 45953) + +# A TCP DNS message which is a query for '. CH A' transaction ID 0xcb37. This is used +# by _wait_until_ready to check that BIND is responding without depending on dnspython. +BIND_TEST_QUERY = bytearray.fromhex("0011cb37000000010000000000000000010003") + + +class DNSServer: + """ + DNSServer configures and handles the lifetime of an RFC2136-capable server. + DNServer provides access to the dns_xdist parameter, listing the address and port + to use for each pytest node. + + At this time, DNSServer should only be used with a single node, but may be expanded in + future to support parallelization (https://github.com/certbot/certbot/issues/8455). + """ + + def __init__(self, unused_nodes, show_output=False): + """ + Create an DNSServer instance. + :param list nodes: list of node names that will be setup by pytest xdist + :param bool show_output: if True, print the output of the DNS server + """ + + self.bind_root = tempfile.mkdtemp() + + self.process = None # type: subprocess.Popen + + self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]} + + # Unfortunately the BIND9 image forces everything to stderr with -g and we can't + # modify the verbosity. + self._output = sys.stderr if show_output else open(os.devnull, "w") + + def start(self): + """Start the DNS server""" + try: + self._configure_bind() + self._start_bind() + except: + self.stop() + raise + + def stop(self): + """Stop the DNS server, and clean its resources""" + if self.process: + try: + self.process.terminate() + self.process.wait() + except BaseException as e: + print("BIND9 did not stop cleanly: {}".format(e), file=sys.stderr) + + shutil.rmtree(self.bind_root, ignore_errors=True) + + if self._output != sys.stderr: + self._output.close() + + def _configure_bind(self): + """Configure the BIND9 server based on the prebaked configuration""" + bind_conf_src = resource_filename( + "certbot_integration_tests", "assets/bind-config" + ) + for directory in ("conf", "zones"): + shutil.copytree( + os.path.join(bind_conf_src, directory), os.path.join(self.bind_root, directory) + ) + + def _start_bind(self): + """Launch the BIND9 server as a Docker container""" + addr_str = "{}:{}".format(BIND_BIND_ADDRESS[0], BIND_BIND_ADDRESS[1]) + self.process = subprocess.Popen( + [ + "docker", + "run", + "--rm", + "-p", + "{}:53/udp".format(addr_str), + "-p", + "{}:53/tcp".format(addr_str), + "-v", + "{}/conf:/etc/bind".format(self.bind_root), + "-v", + "{}/zones:/var/lib/bind".format(self.bind_root), + BIND_DOCKER_IMAGE, + ], + stdout=self._output, + stderr=self._output, + ) + + if self.process.poll(): + raise ValueError("BIND9 server stopped unexpectedly") + + try: + self._wait_until_ready() + except: + # The container might be running even if we think it isn't + self.stop() + raise + + def _wait_until_ready(self, attempts=30): + # type: (int) -> None + """ + Polls the DNS server over TCP until it gets a response, or until + it runs out of attempts and raises a ValueError. + The DNS response message must match the txn_id of the DNS query message, + but otherwise the contents are ignored. + :param int attempts: The number of attempts to make. + """ + for _ in range(attempts): + if self.process.poll(): + raise ValueError("BIND9 server stopped unexpectedly") + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(5.0) + try: + sock.connect(BIND_BIND_ADDRESS) + sock.sendall(BIND_TEST_QUERY) + buf = sock.recv(1024) + # We should receive a DNS message with the same tx_id + if buf and len(buf) > 4 and buf[2:4] == BIND_TEST_QUERY[2:4]: + return + # If we got a response but it wasn't the one we wanted, wait a little + time.sleep(1) + except: # pylint: disable=bare-except + # If there was a network error, wait a little + time.sleep(1) + finally: + sock.close() + + raise ValueError( + "Gave up waiting for DNS server {} to respond".format(BIND_BIND_ADDRESS) + ) + + def __enter__(self): + self.start() + return self.dns_xdist + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index 38c2e60a8..cf2e7ff09 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -4,10 +4,12 @@ or outside during setup/teardown of the integration tests environment. """ import contextlib import errno +import http.server as SimpleHTTPServer import multiprocessing import os import re import shutil +import socketserver import stat import sys import tempfile @@ -23,8 +25,6 @@ from cryptography.x509 import load_pem_x509_certificate from OpenSSL import crypto import pkg_resources import requests -from six.moves import SimpleHTTPServer -from six.moves import socketserver from certbot_integration_tests.utils.constants import \ PEBBLE_ALTERNATE_ROOTS, PEBBLE_MANAGEMENT_URL @@ -39,6 +39,7 @@ def _suppress_x509_verification_warnings(): urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) except ImportError: # Handle old versions of request with vendorized urllib3 + # pylint: disable=no-member from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) @@ -256,7 +257,8 @@ def generate_csr(domains, key_path, csr_path, key_type=RSA_KEY_TYPE): def read_certificate(cert_path): """ - Load the certificate from the provided path, and return a human readable version of it (TEXT mode). + Load the certificate from the provided path, and return a human readable version + of it (TEXT mode). :param str cert_path: the path to the certificate :returns: the TEXT version of the certificate, as it would be displayed by openssl binary """ @@ -280,7 +282,11 @@ def load_sample_data_path(workspace): if os.name == 'nt': # Fix the symlinks on Windows if GIT is not configured to create them upon checkout - for lineage in ['a.encryption-example.com', 'b.encryption-example.com']: + for lineage in [ + 'a.encryption-example.com', + 'b.encryption-example.com', + 'c.encryption-example.com', + ]: current_live = os.path.join(copied, 'live', lineage) for name in os.listdir(current_live): if name != 'README': @@ -305,7 +311,7 @@ def echo(keyword, path=None): if not re.match(r'^\w+$', keyword): raise ValueError('Error, keyword `{0}` is not a single keyword.' .format(keyword)) - return '{0} -c "from __future__ import print_function; print(\'{1}\')"{2}'.format( + return '{0} -c "print(\'{1}\')"{2}'.format( os.path.basename(sys.executable), keyword, ' >> "{0}"'.format(path) if path else '') diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index 7fe03b990..cd62e1a7f 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -1,3 +1,5 @@ +# pylint: disable=missing-module-docstring + import json import os import stat @@ -5,18 +7,19 @@ import stat import pkg_resources import requests -from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT +from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT, MOCK_OCSP_SERVER_PORT PEBBLE_VERSION = 'v2.3.0' ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') -def fetch(workspace): +def fetch(workspace, http_01_port=DEFAULT_HTTP_01_PORT): + # pylint: disable=missing-function-docstring suffix = 'linux-amd64' if os.name != 'nt' else 'windows-amd64.exe' pebble_path = _fetch_asset('pebble', suffix) challtestsrv_path = _fetch_asset('pebble-challtestsrv', suffix) - pebble_config_path = _build_pebble_config(workspace) + pebble_config_path = _build_pebble_config(workspace, http_01_port) return pebble_path, challtestsrv_path, pebble_config_path @@ -35,7 +38,7 @@ def _fetch_asset(asset, suffix): return asset_path -def _build_pebble_config(workspace): +def _build_pebble_config(workspace, http_01_port): config_path = os.path.join(workspace, 'pebble-config.json') with open(config_path, 'w') as file_h: file_h.write(json.dumps({ @@ -44,7 +47,7 @@ def _build_pebble_config(workspace): 'managementListenAddress': '0.0.0.0:15000', 'certificate': os.path.join(ASSETS_PATH, 'cert.pem'), 'privateKey': os.path.join(ASSETS_PATH, 'key.pem'), - 'httpPort': 5002, + 'httpPort': http_01_port, 'tlsPort': 5001, 'ocspResponderURL': 'http://127.0.0.1:{0}'.format(MOCK_OCSP_SERVER_PORT), }, diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py index 9458560e8..4db72998f 100755 --- a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py @@ -4,6 +4,7 @@ This runnable module interfaces itself with the Pebble management interface in o to serve a mock OCSP responder during integration tests against Pebble. """ import datetime +import http.server as BaseHTTPServer import re from cryptography import x509 @@ -13,7 +14,6 @@ from cryptography.hazmat.primitives import serialization from cryptography.x509 import ocsp from dateutil import parser import requests -from six.moves import BaseHTTPServer from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL @@ -21,6 +21,7 @@ from certbot_integration_tests.utils.misc import GracefulTCPServer class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): + # pylint: disable=missing-function-docstring def do_POST(self): request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediate-keys/0', verify=False) issuer_key = serialization.load_pem_private_key(request.content, None, default_backend()) @@ -35,20 +36,28 @@ class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len)) response = requests.get('{0}/cert-status-by-serial/{1}'.format( - PEBBLE_MANAGEMENT_URL, str(hex(ocsp_request.serial_number)).replace('0x', '')), verify=False) + PEBBLE_MANAGEMENT_URL, str(hex(ocsp_request.serial_number)).replace('0x', '')), + verify=False + ) if not response.ok: - ocsp_response = ocsp.OCSPResponseBuilder.build_unsuccessful(ocsp.OCSPResponseStatus.UNAUTHORIZED) + ocsp_response = ocsp.OCSPResponseBuilder.build_unsuccessful( + ocsp.OCSPResponseStatus.UNAUTHORIZED + ) else: data = response.json() now = datetime.datetime.utcnow() cert = x509.load_pem_x509_certificate(data['Certificate'].encode(), default_backend()) if data['Status'] != 'Revoked': - ocsp_status, revocation_time, revocation_reason = ocsp.OCSPCertStatus.GOOD, None, None + ocsp_status = ocsp.OCSPCertStatus.GOOD + revocation_time = None + revocation_reason = None else: - ocsp_status, revocation_reason = ocsp.OCSPCertStatus.REVOKED, x509.ReasonFlags.unspecified - revoked_at = re.sub(r'( \+\d{4}).*$', r'\1', data['RevokedAt']) # "... +0000 UTC" => "+0000" + ocsp_status = ocsp.OCSPCertStatus.REVOKED + revocation_reason = x509.ReasonFlags.unspecified + # "... +0000 UTC" => "+0000" + revoked_at = re.sub(r'( \+\d{4}).*$', r'\1', data['RevokedAt']) revocation_time = parser.parse(revoked_at) ocsp_response = ocsp.OCSPResponseBuilder().add_response( diff --git a/certbot-ci/certbot_integration_tests/utils/proxy.py b/certbot-ci/certbot_integration_tests/utils/proxy.py index 3a16adebf..7c46e640d 100644 --- a/certbot-ci/certbot_integration_tests/utils/proxy.py +++ b/certbot-ci/certbot_integration_tests/utils/proxy.py @@ -1,16 +1,20 @@ #!/usr/bin/env python +# pylint: disable=missing-module-docstring + +import http.server as BaseHTTPServer import json import re import sys import requests -from six.moves import BaseHTTPServer from certbot_integration_tests.utils.misc import GracefulTCPServer def _create_proxy(mapping): + # pylint: disable=missing-function-docstring class ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): + # pylint: disable=missing-class-docstring def do_GET(self): headers = {key.lower(): value for key, value in self.headers.items()} backend = [backend for pattern, backend in mapping.items() diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index 971db2b8e..9f9c1f462 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -18,7 +18,6 @@ install_requires = [ 'python-dateutil', 'pyyaml', 'requests', - 'six', ] # Add pywin32 on Windows platforms to handle low-level system calls. @@ -40,18 +39,17 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-ci/windows_installer_integration_tests/conftest.py b/certbot-ci/windows_installer_integration_tests/conftest.py index e36654f90..8a9de057f 100644 --- a/certbot-ci/windows_installer_integration_tests/conftest.py +++ b/certbot-ci/windows_installer_integration_tests/conftest.py @@ -6,10 +6,8 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ -from __future__ import print_function -import os -import pytest +import os ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index f66e4c945..e0a439d01 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -8,11 +8,11 @@ RUN apt-get update && \ WORKDIR /opt/certbot/src # We copy all contents of the build directory to allow us to easily use -# things like tools/venv3.py which expects all of our packages to be available. +# things like tools/venv.py which expects all of our packages to be available. COPY . . -RUN tools/venv3.py -ENV PATH /opt/certbot/src/venv3/bin:$PATH +RUN tools/venv.py +ENV PATH /opt/certbot/src/venv/bin:$PATH # install in editable mode (-e) to save space: it's not possible to # "rm -rf /opt/certbot/src" (it's stays in the underlaying image); diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index b6fbe2817..2b3f94581 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -2,11 +2,8 @@ import os import shutil import subprocess +from unittest import mock -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore import zope.interface from certbot import errors as le_errors diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index 34aa9133f..bf768f8f8 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -11,7 +11,7 @@ from certbot_compatibility_test import util logger = logging.getLogger(__name__) -class Proxy(object): +class Proxy: """A common base for compatibility test configurators""" @classmethod diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index bfa03f22d..226b8585b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -3,8 +3,6 @@ import logging import socket import requests -import six -from six.moves import xrange from acme import crypto_util from acme import errors as acme_errors @@ -12,18 +10,18 @@ from acme import errors as acme_errors logger = logging.getLogger(__name__) -class Validator(object): +class Validator: """Collection of functions to test a live webserver's configuration""" def certificate(self, cert, name, alt_host=None, port=443): """Verifies the certificate presented at name is cert""" if alt_host is None: host = socket.gethostbyname(name).encode() - elif isinstance(alt_host, six.binary_type): + elif isinstance(alt_host, bytes): host = alt_host else: host = alt_host.encode() - name = name if isinstance(name, six.binary_type) else name.encode() + name = name if isinstance(name, bytes) else name.encode() try: presented_cert = crypto_util.probe_sni(name, host, port) @@ -62,7 +60,7 @@ class Validator(object): else: response = requests.get(url, allow_redirects=False) - return response.status_code in xrange(300, 309) + return response.status_code in range(300, 309) def hsts(self, name): """Test for HTTP Strict Transport Security header""" diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator_test.py b/certbot-compatibility-test/certbot_compatibility_test/validator_test.py index 0b1056561..711d1b38e 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator_test.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator_test.py @@ -1,10 +1,7 @@ """Tests for certbot_compatibility_test.validator.""" import unittest +from unittest import mock -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore import OpenSSL import requests diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 598454ae8..3e9e6a7c2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -1,29 +1,17 @@ -from distutils.version import LooseVersion import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' install_requires = [ 'certbot', 'certbot-apache', - 'six', 'requests', 'zope.interface', ] -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - if sys.version_info < (2, 7, 9): # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) install_requires.append('ndg-httpsclient') @@ -38,18 +26,17 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py index d59862a3c..81c053c04 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_cloudflare.dns_cloudflare` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Cloudflare API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index 5978af85c..c896a6e3a 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -85,7 +85,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _CloudflareClient(self.credentials.conf('email'), self.credentials.conf('api-key')) -class _CloudflareClient(object): +class _CloudflareClient: """ Encapsulates all communication with the Cloudflare API. """ diff --git a/certbot-dns-cloudflare/docs/conf.py b/certbot-dns-cloudflare/docs/conf.py index 21c1d9b72..b80bdbc97 100644 --- a/certbot-dns-cloudflare/docs/conf.py +++ b/certbot-dns-cloudflare/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 96db7351a..2e5215f9c 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -1,18 +1,16 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'cloudflare>=1.5.1', - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -49,7 +38,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -57,12 +46,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py index 6ddbdfe5a..0d098445c 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_cloudxns.dns_cloudxns` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the CloudXNS API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-cloudxns/docs/conf.py b/certbot-dns-cloudxns/docs/conf.py index de6f554da..5a350b1b1 100644 --- a/certbot-dns-cloudxns/docs/conf.py +++ b/certbot-dns-cloudxns/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 09225a42e..44bada6bc 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -1,18 +1,16 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -49,7 +38,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -57,12 +46,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py index 3ab8df041..2cb7a92de 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_digitalocean.dns_digitalocean` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the DigitalOcean API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index 75e25a848..5893df9c2 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -19,7 +19,8 @@ class Authenticator(dns_common.DNSAuthenticator): This Authenticator uses the DigitalOcean API to fulfill a dns-01 challenge. """ - description = 'Obtain certs using a DNS TXT record (if you are using DigitalOcean for DNS).' + description = 'Obtain certificates using a DNS TXT record (if you are ' + \ + 'using DigitalOcean for DNS).' def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) @@ -53,7 +54,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _DigitalOceanClient(self.credentials.conf('token')) -class _DigitalOceanClient(object): +class _DigitalOceanClient: """ Encapsulates all communication with the DigitalOcean API. """ diff --git a/certbot-dns-digitalocean/docs/conf.py b/certbot-dns-digitalocean/docs/conf.py index ab653a2b0..5951e3f98 100644 --- a/certbot-dns-digitalocean/docs/conf.py +++ b/certbot-dns-digitalocean/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 4da161146..763961e79 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -1,19 +1,16 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'python-digitalocean>=1.11', - 'setuptools', - 'six', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -28,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -50,7 +38,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -58,12 +46,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py index f8a2e83aa..0f6168a13 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_dnsimple.dns_dnsimple` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the DNSimple API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-dnsimple/docs/conf.py b/certbot-dns-dnsimple/docs/conf.py index 4c6e6b52e..7f88e6387 100644 --- a/certbot-dns-dnsimple/docs/conf.py +++ b/certbot-dns-dnsimple/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index cd1246e07..99a3c92a3 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -1,17 +1,15 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -26,15 +24,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - # This package normally depends on dns-lexicon>=3.2.1 to address the # problem described in https://github.com/AnalogJ/lexicon/issues/387, # however, the fix there has been backported to older versions of @@ -60,7 +49,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -68,12 +57,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/__init__.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/__init__.py index 52f055237..fa49ee516 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/__init__.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_dnsmadeeasy.dns_dnsmadeeasy` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the DNS Made Easy API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-dnsmadeeasy/docs/conf.py b/certbot-dns-dnsmadeeasy/docs/conf.py index 1dfc1bd89..efe2f36f4 100644 --- a/certbot-dns-dnsmadeeasy/docs/conf.py +++ b/certbot-dns-dnsmadeeasy/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index c077acbef..b126b4e8f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -1,18 +1,16 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -49,7 +38,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -57,12 +46,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py b/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py index fdcb8cd48..fd81d0712 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_gehirn.dns_gehirn` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Gehirn Infrastructure Service DNS API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-gehirn/docs/conf.py b/certbot-dns-gehirn/docs/conf.py index 75e3705dd..2cc968fe0 100644 --- a/certbot-dns-gehirn/docs/conf.py +++ b/certbot-dns-gehirn/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index ba58509f7..cada7a85e 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -1,17 +1,15 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ 'dns-lexicon>=2.1.22', - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -26,15 +24,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -48,7 +37,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -56,12 +45,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py index b88260b07..2d448c590 100644 --- a/certbot-dns-google/certbot_dns_google/__init__.py +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_google.dns_google` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Google Cloud DNS API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index 4eaed1783..363c5e079 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -76,7 +76,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _GoogleClient(self.conf('credentials')) -class _GoogleClient(object): +class _GoogleClient: """ Encapsulates all communication with the Google Cloud DNS API. """ @@ -85,9 +85,13 @@ class _GoogleClient(object): scopes = ['https://www.googleapis.com/auth/ndev.clouddns.readwrite'] if account_json is not None: - credentials = ServiceAccountCredentials.from_json_keyfile_name(account_json, scopes) - with open(account_json) as account: - self.project_id = json.load(account)['project_id'] + try: + credentials = ServiceAccountCredentials.from_json_keyfile_name(account_json, scopes) + with open(account_json) as account: + self.project_id = json.load(account)['project_id'] + except Exception as e: + raise errors.PluginError( + "Error parsing credentials file '{}': {}".format(account_json, e)) else: credentials = None self.project_id = self.get_project_id() @@ -114,10 +118,13 @@ class _GoogleClient(object): record_contents = self.get_existing_txt_rrset(zone_id, record_name) if record_contents is None: - record_contents = [] - add_records = record_contents[:] + # If it wasn't possible to fetch the records at this label (missing .list permission), + # assume there aren't any (#5678). If there are actually records here, this will fail + # with HTTP 409/412 API errors. + record_contents = {"rrdatas": []} + add_records = record_contents["rrdatas"][:] - if "\""+record_content+"\"" in record_contents: + if "\""+record_content+"\"" in record_contents["rrdatas"]: # The process was interrupted previously and validation token exists return @@ -136,15 +143,15 @@ class _GoogleClient(object): ], } - if record_contents: + if record_contents["rrdatas"]: # We need to remove old records in the same request data["deletions"] = [ { "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": record_contents, - "ttl": record_ttl, + "rrdatas": record_contents["rrdatas"], + "ttl": record_contents["ttl"], }, ] @@ -184,7 +191,10 @@ class _GoogleClient(object): record_contents = self.get_existing_txt_rrset(zone_id, record_name) if record_contents is None: - record_contents = ["\"" + record_content + "\""] + # If it wasn't possible to fetch the records at this label (missing .list permission), + # assume there aren't any (#5678). If there are actually records here, this will fail + # with HTTP 409/412 API errors. + record_contents = {"rrdatas": ["\"" + record_content + "\""], "ttl": record_ttl} data = { "kind": "dns#change", @@ -193,14 +203,15 @@ class _GoogleClient(object): "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": record_contents, - "ttl": record_ttl, + "rrdatas": record_contents["rrdatas"], + "ttl": record_contents["ttl"], }, ], } # Remove the record being deleted from the list - readd_contents = [r for r in record_contents if r != "\"" + record_content + "\""] + readd_contents = [r for r in record_contents["rrdatas"] + if r != "\"" + record_content + "\""] if readd_contents: # We need to remove old records in the same request data["additions"] = [ @@ -209,7 +220,7 @@ class _GoogleClient(object): "type": "TXT", "name": record_name + ".", "rrdatas": readd_contents, - "ttl": record_ttl, + "ttl": record_contents["ttl"], }, ] @@ -231,14 +242,15 @@ class _GoogleClient(object): :param str zone_id: The ID of the managed zone. :param str record_name: The record name (typically beginning with '_acme-challenge.'). - :returns: List of TXT record values or None - :rtype: `list` of `string` or `None` + :returns: The resourceRecordSet corresponding to `record_name` or None + :rtype: `resourceRecordSet ` or `None` # pylint: disable=line-too-long """ rrs_request = self.dns.resourceRecordSets() - request = rrs_request.list(managedZone=zone_id, project=self.project_id) # Add dot as the API returns absolute domains record_name += "." + request = rrs_request.list(project=self.project_id, managedZone=zone_id, name=record_name, + type="TXT") try: response = request.execute() except googleapiclient_errors.Error: @@ -246,10 +258,8 @@ class _GoogleClient(object): "requesting a wildcard certificate, this might not work.") logger.debug("Error was:", exc_info=True) else: - if response: - for rr in response["rrsets"]: - if rr["name"] == record_name and rr["type"] == "TXT": - return rr["rrdatas"] + if response and response["rrsets"]: + return response["rrsets"][0] return None def _find_managed_zone_id(self, domain): diff --git a/certbot-dns-google/docs/conf.py b/certbot-dns-google/docs/conf.py index 8c4a800f7..06bb99f46 100644 --- a/certbot-dns-google/docs/conf.py +++ b/certbot-dns-google/docs/conf.py @@ -112,7 +112,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 78a78cbc3..e8b26eceb 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -1,19 +1,17 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'google-api-python-client>=1.5.5', 'oauth2client>=4.0', - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', # already a dependency of google-api-python-client, but added for consistency 'httplib2' @@ -30,15 +28,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -52,7 +41,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -60,12 +49,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 5af027cef..7de5f1d67 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -70,7 +70,7 @@ class GoogleClientTest(unittest.TestCase): zone = "ZONE_ID" change = "an-id" - def _setUp_client_with_mock(self, zone_request_side_effect): + def _setUp_client_with_mock(self, zone_request_side_effect, rrs_list_side_effect=None): from certbot_dns_google._internal.dns_google import _GoogleClient pwd = os.path.dirname(__file__) @@ -86,9 +86,16 @@ class GoogleClientTest(unittest.TestCase): mock_mz.list.return_value.execute.side_effect = zone_request_side_effect mock_rrs = mock.MagicMock() - rrsets = {"rrsets": [{"name": "_acme-challenge.example.org.", "type": "TXT", - "rrdatas": ["\"example-txt-contents\""]}]} - mock_rrs.list.return_value.execute.return_value = rrsets + def rrs_list(project=None, managedZone=None, name=None, type=None): + response = {"rrsets": []} + if name == "_acme-challenge.example.org.": + response = {"rrsets": [{"name": "_acme-challenge.example.org.", "type": "TXT", + "rrdatas": ["\"example-txt-contents\""], "ttl": 60}]} + mock_return = mock.MagicMock() + mock_return.execute.return_value = response + mock_return.execute.side_effect = rrs_list_side_effect + return mock_return + mock_rrs.list.side_effect = rrs_list mock_changes = mock.MagicMock() client.dns.managedZones = mock.MagicMock(return_value=mock_mz) @@ -107,6 +114,17 @@ class GoogleClientTest(unittest.TestCase): self.assertFalse(credential_mock.called) self.assertTrue(get_project_id_mock.called) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + def test_client_bad_credentials_file(self, credential_mock): + credential_mock.side_effect = ValueError('Some exception buried in oauth2client') + with self.assertRaises(errors.PluginError) as cm: + self._setUp_client_with_mock([]) + self.assertEqual( + str(cm.exception), + "Error parsing credentials file '/not/a/real/path.json': " + "Some exception buried in oauth2client" + ) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) @@ -162,11 +180,29 @@ class GoogleClientTest(unittest.TestCase): # pylint: disable=line-too-long mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" with mock.patch(mock_get_rrs) as mock_rrs: - mock_rrs.return_value = ["sample-txt-contents"] + mock_rrs.return_value = {"rrdatas": ["sample-txt-contents"], "ttl": self.record_ttl} client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) self.assertTrue(changes.create.called) - self.assertTrue("sample-txt-contents" in - changes.create.call_args_list[0][1]["body"]["deletions"][0]["rrdatas"]) + deletions = changes.create.call_args_list[0][1]["body"]["deletions"][0] + self.assertTrue("sample-txt-contents" in deletions["rrdatas"]) + self.assertEqual(self.record_ttl, deletions["ttl"]) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google._internal.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_delete_old_ttl_case(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + # pylint: disable=line-too-long + mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + custom_ttl = 300 + mock_rrs.return_value = {"rrdatas": ["sample-txt-contents"], "ttl": custom_ttl} + client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + self.assertTrue(changes.create.called) + deletions = changes.create.call_args_list[0][1]["body"]["deletions"][0] + self.assertTrue("sample-txt-contents" in deletions["rrdatas"]) + self.assertEqual(custom_ttl, deletions["ttl"]) #otherwise HTTP 412 @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', @@ -210,14 +246,13 @@ class GoogleClientTest(unittest.TestCase): @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) - def test_del_txt_record(self, unused_credential_mock): + def test_del_txt_record_multi_rrdatas(self, unused_credential_mock): client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) - # pylint: disable=line-too-long mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" with mock.patch(mock_get_rrs) as mock_rrs: - mock_rrs.return_value = ["\"sample-txt-contents\"", - "\"example-txt-contents\""] + mock_rrs.return_value = {"rrdatas": ["\"sample-txt-contents\"", + "\"example-txt-contents\""], "ttl": self.record_ttl} client.del_txt_record(DOMAIN, "_acme-challenge.example.org", "example-txt-contents", self.record_ttl) @@ -250,19 +285,48 @@ class GoogleClientTest(unittest.TestCase): @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) - def test_del_txt_record_error_during_zone_lookup(self, unused_credential_mock): - client, unused_changes = self._setUp_client_with_mock(API_ERROR) + def test_del_txt_record_single_rrdatas(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + # pylint: disable=line-too-long + mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + mock_rrs.return_value = {"rrdatas": ["\"example-txt-contents\""], "ttl": self.record_ttl} + client.del_txt_record(DOMAIN, "_acme-challenge.example.org", + "example-txt-contents", self.record_ttl) + expected_body = { + "kind": "dns#change", + "deletions": [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": "_acme-challenge.example.org.", + "rrdatas": ["\"example-txt-contents\""], + "ttl": self.record_ttl, + }, + ], + } + + changes.create.assert_called_with(body=expected_body, + managedZone=self.zone, + project=PROJECT_ID) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google._internal.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_del_txt_record_error_during_zone_lookup(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock(API_ERROR) client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + changes.create.assert_not_called() @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) def test_del_txt_record_zone_not_found(self, unused_credential_mock): - client, unused_changes = self._setUp_client_with_mock([{'managedZones': []}, + client, changes = self._setUp_client_with_mock([{'managedZones': []}, {'managedZones': []}]) - client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + changes.create.assert_not_called() @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', @@ -276,24 +340,39 @@ class GoogleClientTest(unittest.TestCase): @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) - def test_get_existing(self, unused_credential_mock): + def test_get_existing_found(self, unused_credential_mock): client, unused_changes = self._setUp_client_with_mock( [{'managedZones': [{'id': self.zone}]}]) # Record name mocked in setUp found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") - self.assertEqual(found, ["\"example-txt-contents\""]) + self.assertEqual(found["rrdatas"], ["\"example-txt-contents\""]) + self.assertEqual(found["ttl"], 60) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google._internal.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing_not_found(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") self.assertEqual(not_found, None) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google._internal.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing_with_error(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}], API_ERROR) + # Record name mocked in setUp + found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") + self.assertEqual(found, None) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) def test_get_existing_fallback(self, unused_credential_mock): client, unused_changes = self._setUp_client_with_mock( - [{'managedZones': [{'id': self.zone}]}]) - mock_execute = client.dns.resourceRecordSets.return_value.list.return_value.execute - mock_execute.side_effect = API_ERROR - + [{'managedZones': [{'id': self.zone}]}], API_ERROR) rrset = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") self.assertFalse(rrset) @@ -322,7 +401,7 @@ class GoogleClientTest(unittest.TestCase): self.assertRaises(ServerNotFoundError, _GoogleClient.get_project_id) -class DummyResponse(object): +class DummyResponse: """ Dummy object to create a fake HTTPResponse (the actual one requires a socket and we only need the status attribute) diff --git a/certbot-dns-linode/certbot_dns_linode/__init__.py b/certbot-dns-linode/certbot_dns_linode/__init__.py index 4bfd95573..bca15bdb2 100644 --- a/certbot-dns-linode/certbot_dns_linode/__init__.py +++ b/certbot-dns-linode/certbot_dns_linode/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_linode.dns_linode` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Linode API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py index f9450c02c..c1b5e066f 100644 --- a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py @@ -24,7 +24,7 @@ class Authenticator(dns_common.DNSAuthenticator): This Authenticator uses the Linode API to fulfill a dns-01 challenge. """ - description = 'Obtain certs using a DNS TXT record (if you are using Linode for DNS).' + description = 'Obtain certificates using a DNS TXT record (if you are using Linode for DNS).' def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) diff --git a/certbot-dns-linode/docs/conf.py b/certbot-dns-linode/docs/conf.py index 6305b694c..c916b5097 100644 --- a/certbot-dns-linode/docs/conf.py +++ b/certbot-dns-linode/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 4be13cf56..63da31df2 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,17 +1,15 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ 'dns-lexicon>=2.2.3', - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -26,15 +24,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -48,7 +37,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -56,12 +45,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-luadns/certbot_dns_luadns/__init__.py b/certbot-dns-luadns/certbot_dns_luadns/__init__.py index e8e86f77c..302cb1392 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/__init__.py +++ b/certbot-dns-luadns/certbot_dns_luadns/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_luadns.dns_luadns` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the LuaDNS API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-luadns/docs/conf.py b/certbot-dns-luadns/docs/conf.py index 6a11ce7aa..5790a85a7 100644 --- a/certbot-dns-luadns/docs/conf.py +++ b/certbot-dns-luadns/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 805158771..61b9dfa78 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -1,18 +1,16 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -49,7 +38,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -57,12 +46,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-nsone/certbot_dns_nsone/__init__.py b/certbot-dns-nsone/certbot_dns_nsone/__init__.py index e59be74a7..6c7d41ba4 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/__init__.py +++ b/certbot-dns-nsone/certbot_dns_nsone/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_nsone.dns_nsone` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the NS1 API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-nsone/docs/conf.py b/certbot-dns-nsone/docs/conf.py index 7e66a0613..4bc13fe2b 100644 --- a/certbot-dns-nsone/docs/conf.py +++ b/certbot-dns-nsone/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 100a48551..d60412f27 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -1,18 +1,16 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -49,7 +38,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -57,12 +46,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-ovh/certbot_dns_ovh/__init__.py b/certbot-dns-ovh/certbot_dns_ovh/__init__.py index d508fad1b..6a079e59f 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/__init__.py +++ b/certbot-dns-ovh/certbot_dns_ovh/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_ovh.dns_ovh` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the OVH API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-ovh/docs/conf.py b/certbot-dns-ovh/docs/conf.py index c8a1575c4..18ebccaac 100644 --- a/certbot-dns-ovh/docs/conf.py +++ b/certbot-dns-ovh/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 585b038a3..4d6131ff7 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -1,18 +1,16 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -49,7 +38,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -57,12 +46,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py index da8ef3419..3c574835f 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_rfc2136.dns_rfc2136` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using RFC 2136 Dynamic Updates. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py index 57e9506f2..adebdd963 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py @@ -88,7 +88,7 @@ class Authenticator(dns_common.DNSAuthenticator): dns.tsig.HMAC_MD5)) -class _RFC2136Client(object): +class _RFC2136Client: """ Encapsulates all communication with the target DNS server. """ diff --git a/certbot-dns-rfc2136/docs/conf.py b/certbot-dns-rfc2136/docs/conf.py index bc0e9c845..782f494f1 100644 --- a/certbot-dns-rfc2136/docs/conf.py +++ b/certbot-dns-rfc2136/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index c841aefc1..18d006732 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -1,18 +1,16 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'dnspython', - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -49,7 +38,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -57,12 +46,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py index de5e2912b..dc4a73a04 100644 --- a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py @@ -154,7 +154,7 @@ class RFC2136ClientTest(unittest.TestCase): # _find_domain | pylint: disable=protected-access domain = self.rfc2136_client._find_domain('foo.bar.'+DOMAIN) - self.assertTrue(domain == DOMAIN) + self.assertEqual(domain, DOMAIN) def test_find_domain_wraps_errors(self): # _query_soa | pylint: disable=protected-access diff --git a/certbot-dns-route53/MANIFEST.in b/certbot-dns-route53/MANIFEST.in index fc62028b0..5a661cef6 100644 --- a/certbot-dns-route53/MANIFEST.in +++ b/certbot-dns-route53/MANIFEST.in @@ -1,5 +1,5 @@ include LICENSE.txt -include README +include README.rst recursive-include docs * recursive-include tests * global-exclude __pycache__ diff --git a/certbot-dns-route53/README.md b/certbot-dns-route53/README.md deleted file mode 100644 index 4af66aa00..000000000 --- a/certbot-dns-route53/README.md +++ /dev/null @@ -1,35 +0,0 @@ -## Route53 plugin for Let's Encrypt client - -### Before you start - -It's expected that the root hosted zone for the domain in question already -exists in your account. - -### Setup - -1. Create a virtual environment - -2. Update its pip and setuptools (`VENV/bin/pip install -U setuptools pip`) -to avoid problems with cryptography's dependency on setuptools>=11.3. - -3. Make sure you have libssl-dev and libffi (or your regional equivalents) -installed. You might have to set compiler flags to pick things up (I have to -use `CPPFLAGS=-I/usr/local/opt/openssl/include -LDFLAGS=-L/usr/local/opt/openssl/lib` on my macOS to pick up brew's openssl, -for example). - -4. Install this package. - -### How to use it - -Make sure you have access to AWS's Route53 service, either through IAM roles or -via `.aws/credentials`. Check out -[sample-aws-policy.json](examples/sample-aws-policy.json) for the necessary permissions. - -To generate a certificate: -``` -certbot certonly \ - -n --agree-tos --email DEVOPS@COMPANY.COM \ - --dns-route53 \ - -d MY.DOMAIN.NAME -``` diff --git a/certbot-dns-route53/README.rst b/certbot-dns-route53/README.rst new file mode 100644 index 000000000..cf63762ca --- /dev/null +++ b/certbot-dns-route53/README.rst @@ -0,0 +1 @@ +Amazon Web Services Route 53 DNS Authenticator plugin for Certbot diff --git a/certbot-dns-route53/certbot_dns_route53/__init__.py b/certbot-dns-route53/certbot_dns_route53/__init__.py index 8659617ef..1b59f5620 100644 --- a/certbot-dns-route53/certbot_dns_route53/__init__.py +++ b/certbot-dns-route53/certbot_dns_route53/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_route53.dns_route53` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Amazon Web Services Route 53 API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-route53/docs/conf.py b/certbot-dns-route53/docs/conf.py index f9c5fdf74..bb7808d66 100644 --- a/certbot-dns-route53/docs/conf.py +++ b/certbot-dns-route53/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 2ad0de045..fdc931bbe 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,18 +1,16 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'boto3', - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -27,14 +25,10 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] setup( name='certbot-dns-route53', @@ -44,7 +38,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -52,12 +46,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', @@ -69,6 +62,9 @@ setup( include_package_data=True, install_requires=install_requires, keywords=['certbot', 'route53', 'aws'], + extras_require={ + 'docs': docs_extras, + }, entry_points={ 'certbot.plugins': [ 'dns-route53 = certbot_dns_route53._internal.dns_route53:Authenticator', diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py index f18780c18..c16ee96ef 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_sakuracloud.dns_sakuracloud` plugin automates the process of c a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Sakura Cloud DNS API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-sakuracloud/docs/conf.py b/certbot-dns-sakuracloud/docs/conf.py index 9d6d3c871..b84b97e1d 100644 --- a/certbot-dns-sakuracloud/docs/conf.py +++ b/certbot-dns-sakuracloud/docs/conf.py @@ -111,7 +111,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index b1c6ee6a3..55f140d76 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -1,17 +1,15 @@ -from distutils.version import LooseVersion import os import sys -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ 'dns-lexicon>=2.1.23', - 'setuptools', + 'setuptools>=39.0.1', 'zope.interface', ] @@ -26,15 +24,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', @@ -48,7 +37,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -56,12 +45,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 87afedd38..8a3b8078f 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-lines """Nginx Configuration""" from distutils.version import LooseVersion import logging @@ -15,8 +16,10 @@ from acme import challenges from acme import crypto_util as acme_crypto_util from acme.magic_typing import Dict from acme.magic_typing import List +from acme.magic_typing import Optional from acme.magic_typing import Set from acme.magic_typing import Text +from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -105,7 +108,7 @@ class NginxConfigurator(common.Installer): self.save_notes = "" # For creating new vhosts if no names match - self.new_vhost = None + self.new_vhost: Optional[obj.VirtualHost] = None # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() @@ -116,7 +119,7 @@ class NginxConfigurator(common.Installer): self._chall_out = 0 # These will be set in the prepare function - self.parser = None + self.parser: Optional[parser.NginxParser] = None self.version = version self.openssl_version = openssl_version self._enhance_func = {"redirect": self._enable_redirect, @@ -226,7 +229,7 @@ class NginxConfigurator(common.Installer): if not fullchain_path: raise errors.PluginError( "The nginx plugin currently requires --fullchain-path to " - "install a cert.") + "install a certificate.") vhosts = self.choose_vhosts(domain, create_if_no_match=True) for vhost in vhosts: @@ -377,10 +380,13 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain, allow_port_mismatch, port): + def _vhost_from_duplicated_default(self, domain: str, allow_port_mismatch: bool, port: str + ) -> obj.VirtualHost: """if allow_port_mismatch is False, only server blocks with matching ports will be used as a default server block template. """ + assert self.parser is not None # prepare should already have been called here + if self.new_vhost is None: default_vhost = self._get_default_vhost(domain, allow_port_mismatch, port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, @@ -509,7 +515,7 @@ class NginxConfigurator(common.Installer): match['rank'] += NO_SSL_MODIFIER return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): + def choose_redirect_vhosts(self, target_name: str, port: str) -> List[obj.VirtualHost]: """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -523,9 +529,6 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param str port: port number - :param bool create_if_no_match: If we should create a new vhost from default - when there is no match found. If we can't choose a default, raise a - MisconfigurationError. :returns: vhosts associated with name :rtype: list of :class:`~certbot_nginx._internal.obj.VirtualHost` @@ -538,32 +541,75 @@ class NginxConfigurator(common.Installer): else: matches = self._get_redirect_ranked_matches(target_name, port) vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] - if not vhosts and create_if_no_match: - vhosts = [self._vhost_from_duplicated_default(target_name, False, port)] return vhosts - def _port_matches(self, test_port, matching_port): + def choose_auth_vhosts(self, target_name: str) -> Tuple[List[obj.VirtualHost], + List[obj.VirtualHost]]: + """Returns a list of HTTP and HTTPS vhosts with a server_name matching target_name. + + If no HTTP vhost exists, one will be cloned from the default vhost. If that fails, no HTTP + vhost will be returned. + + :param str target_name: non-wildcard domain name + + :returns: tuple of HTTP and HTTPS virtualhosts + :rtype: tuple of :class:`~certbot_nginx._internal.obj.VirtualHost` + + """ + vhosts = [m['vhost'] for m in self._get_ranked_matches(target_name) if m and 'vhost' in m] + http_vhosts = [vh for vh in vhosts if + self._vhost_listening(vh, str(self.config.http01_port), False)] + https_vhosts = [vh for vh in vhosts if + self._vhost_listening(vh, str(self.config.https_port), True)] + + # If no HTTP vhost matches, try create one from the default_server on http01_port. + if not http_vhosts: + try: + http_vhosts = [self._vhost_from_duplicated_default(target_name, False, + str(self.config.http01_port))] + except errors.MisconfigurationError: + http_vhosts = [] + + return http_vhosts, https_vhosts + + def _port_matches(self, test_port: str, matching_port: str) -> bool: # test_port is a number, matching is a number or "" or None if matching_port == "" or matching_port is None: # if no port is specified, Nginx defaults to listening on port 80. return test_port == self.DEFAULT_LISTEN_PORT return test_port == matching_port - def _vhost_listening_on_port_no_ssl(self, vhost, port): - found_matching_port = False - if not vhost.addrs: - # if there are no listen directives at all, Nginx defaults to - # listening on port 80. - found_matching_port = (port == self.DEFAULT_LISTEN_PORT) - else: - for addr in vhost.addrs: - if self._port_matches(port, addr.get_port()) and not addr.ssl: - found_matching_port = True + def _vhost_listening(self, vhost: obj.VirtualHost, port: str, ssl: bool) -> bool: + """Tests whether a vhost has an address listening on a port with SSL enabled or disabled. - if found_matching_port: - # make sure we don't have an 'ssl on' directive - return not self.parser.has_ssl_on_directive(vhost) - return False + :param `obj.VirtualHost` vhost: The vhost whose addresses will be tested + :param port str: The port number as a string that the address should be bound to + :param bool ssl: Whether SSL should be enabled or disabled on the address + + :returns: Whether the vhost has an address listening on the port and protocol. + :rtype: bool + + """ + assert self.parser is not None # prepare should already have been called here + + # if the 'ssl on' directive is present on the vhost, all its addresses have SSL enabled + all_addrs_are_ssl = self.parser.has_ssl_on_directive(vhost) + + # if we want ssl vhosts: either 'ssl on' or 'addr.ssl' should be enabled + # if we want plaintext vhosts: neither 'ssl on' nor 'addr.ssl' should be enabled + _ssl_matches = lambda addr: addr.ssl or all_addrs_are_ssl if ssl else \ + not addr.ssl and not all_addrs_are_ssl + + # if there are no listen directives at all, Nginx defaults to + # listening on port 80. + if not vhost.addrs: + return port == self.DEFAULT_LISTEN_PORT and ssl == all_addrs_are_ssl + + return any(self._port_matches(port, addr.get_port()) and _ssl_matches(addr) + for addr in vhost.addrs) + + def _vhost_listening_on_port_no_ssl(self, vhost: obj.VirtualHost, port: str) -> bool: + return self._vhost_listening(vhost, port, False) def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index 2f6458f87..896760fc4 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -1,9 +1,12 @@ """A class that performs HTTP-01 challenges for Nginx""" +import io import logging from acme import challenges from acme.magic_typing import List +from acme.magic_typing import Optional +from certbot import achallenges from certbot import errors from certbot.compat import os from certbot.plugins import common @@ -102,7 +105,7 @@ class NginxHttp01(common.ChallengePerformer): self.configurator.reverter.register_file_creation( True, self.challenge_conf) - with open(self.challenge_conf, "w") as new_conf: + with io.open(self.challenge_conf, "w", encoding="utf-8") as new_conf: nginxparser.dump(config, new_conf) def _default_listen_addresses(self): @@ -137,13 +140,12 @@ class NginxHttp01(common.ChallengePerformer): def _get_validation_path(self, achall): return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token")) - def _make_server_block(self, achall): + def _make_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge) -> List: """Creates a server block for a challenge. + :param achall: Annotated HTTP-01 challenge - :type achall: - :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` - :param list addrs: addresses of challenged domain - :class:`list` of type :class:`~nginx.obj.Addr` + :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :returns: server block for the challenge host :rtype: list """ @@ -171,34 +173,35 @@ class NginxHttp01(common.ChallengePerformer): return location_directive - def _make_or_mod_server_block(self, achall): - """Modifies a server block to respond to a challenge. + def _make_or_mod_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge + ) -> Optional[List]: + """Modifies server blocks to respond to a challenge. Returns a new HTTP server block + to add to the configuration if an existing one can't be found. :param achall: Annotated HTTP-01 challenge - :type achall: - :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + + :returns: new server block to be added, if any + :rtype: list """ - try: - vhosts = self.configurator.choose_redirect_vhosts(achall.domain, - '%i' % self.configurator.config.http01_port, create_if_no_match=True) - except errors.MisconfigurationError: + http_vhosts, https_vhosts = self.configurator.choose_auth_vhosts(achall.domain) + + new_vhost: Optional[list] = None + if not http_vhosts: # Couldn't find either a matching name+port server block # or a port+default_server block, so create a dummy block - return self._make_server_block(achall) + new_vhost = self._make_server_block(achall) - # len is max 1 because Nginx doesn't authenticate wildcards - # if len were or vhosts None, we would have errored - vhost = vhosts[0] + # Modify any existing server blocks + for vhost in set(http_vhosts + https_vhosts): + location_directive = [self._location_directive_for_achall(achall)] - # Modify existing server block - location_directive = [self._location_directive_for_achall(achall)] + self.configurator.parser.add_server_directives(vhost, location_directive) - self.configurator.parser.add_server_directives(vhost, - location_directive) + rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', + ' ', '$1', ' ', 'break']] + self.configurator.parser.add_server_directives(vhost, + rewrite_directive, insert_at_top=True) - rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', - ' ', '$1', ' ', 'break']] - self.configurator.parser.add_server_directives(vhost, - rewrite_directive, insert_at_top=True) - return None + return new_vhost diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index a8ac90427..a51302fae 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -15,11 +15,11 @@ from pyparsing import restOfLine from pyparsing import stringEnd from pyparsing import White from pyparsing import ZeroOrMore -import six +from acme.magic_typing import IO, Any # pylint: disable=unused-import logger = logging.getLogger(__name__) -class RawNginxParser(object): +class RawNginxParser: # pylint: disable=pointless-statement """A class that parses nginx configuration with pyparsing.""" @@ -69,7 +69,7 @@ class RawNginxParser(object): """Returns the parsed tree as a list.""" return self.parse().asList() -class RawNginxDumper(object): +class RawNginxDumper: """A class that dumps nginx configuration from the provided tree.""" def __init__(self, blocks): self.blocks = blocks @@ -78,7 +78,7 @@ class RawNginxDumper(object): """Iterates the dumped nginx content.""" blocks = blocks or self.blocks for b0 in blocks: - if isinstance(b0, six.string_types): + if isinstance(b0, str): yield b0 continue item = copy.deepcopy(b0) @@ -95,7 +95,7 @@ class RawNginxDumper(object): yield '}' else: # not a block - list of strings semicolon = ";" - if isinstance(item[0], six.string_types) and item[0].strip() == '#': # comment + if isinstance(item[0], str) and item[0].strip() == '#': # comment semicolon = "" yield "".join(item) + semicolon @@ -130,10 +130,10 @@ def load(_file): def dumps(blocks): - """Dump to a string. + # type: (UnspacedList) -> str + """Dump to a Unicode string. :param UnspacedList block: The parsed tree - :param int indentation: The number of spaces to indent :rtype: str """ @@ -141,18 +141,19 @@ def dumps(blocks): def dump(blocks, _file): + # type: (UnspacedList, IO[Any]) -> None """Dump to a file. :param UnspacedList block: The parsed tree - :param file _file: The file to dump to - :param int indentation: The number of spaces to indent - :rtype: NoneType + :param IO[Any] _file: The file stream to dump to. It must be opened with + Unicode encoding. + :rtype: None """ - return _file.write(dumps(blocks)) + _file.write(dumps(blocks)) -spacey = lambda x: (isinstance(x, six.string_types) and x.isspace()) or x == '' +spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == '' class UnspacedList(list): """Wrap a list [of lists], making any whitespace entries magically invisible""" diff --git a/certbot-nginx/certbot_nginx/_internal/obj.py b/certbot-nginx/certbot_nginx/_internal/obj.py index 4e0d8cf35..1511cba6d 100644 --- a/certbot-nginx/certbot_nginx/_internal/obj.py +++ b/certbot-nginx/certbot_nginx/_internal/obj.py @@ -1,7 +1,6 @@ """Module contains classes used by the Nginx Configurator.""" import re -import six from certbot.plugins import common @@ -144,7 +143,7 @@ class Addr(common.Addr): return False -class VirtualHost(object): +class VirtualHost: """Represents an Nginx Virtualhost. :ivar str filep: file path of VH @@ -211,7 +210,7 @@ class VirtualHost(object): def contains_list(self, test): """Determine if raw server block contains test list at top level """ - for i in six.moves.range(0, len(self.raw) - len(test) + 1): + for i in range(0, len(self.raw) - len(test) + 1): if self.raw[i:i + len(test)] == test: return True return False @@ -250,7 +249,7 @@ def _find_directive(directives, directive_name, match_content=None): """Find a directive of type directive_name in directives. If match_content is given, Searches for `match_content` in the directive arguments. """ - if not directives or isinstance(directives, six.string_types): + if not directives or isinstance(directives, str): return None # If match_content is None, just match on directive type. Otherwise, match on diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 641ffb020..db9530104 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -7,10 +7,10 @@ import logging import re import pyparsing -import six from acme.magic_typing import Dict from acme.magic_typing import List +from acme.magic_typing import Optional from acme.magic_typing import Set from acme.magic_typing import Tuple from acme.magic_typing import Union @@ -22,7 +22,7 @@ from certbot_nginx._internal import obj logger = logging.getLogger(__name__) -class NginxParser(object): +class NginxParser: """Class handles the fine details of parsing the Nginx Configuration. :ivar str root: Normalized absolute path to the server root @@ -249,7 +249,7 @@ class NginxParser(object): continue out = nginxparser.dumps(tree) logger.debug('Writing nginx conf tree to %s:\n%s', filename, out) - with open(filename, 'w') as _file: + with io.open(filename, 'w', encoding='utf-8') as _file: _file.write(out) except IOError: @@ -362,8 +362,9 @@ class NginxParser(object): except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) - def duplicate_vhost(self, vhost_template, remove_singleton_listen_params=False, - only_directives=None): + def duplicate_vhost(self, vhost_template: obj.VirtualHost, + remove_singleton_listen_params: bool = False, + only_directives: Optional[List] = None) -> obj.VirtualHost: """Duplicate the vhost in the configuration files. :param :class:`~certbot_nginx._internal.obj.VirtualHost` vhost_template: The vhost @@ -549,7 +550,7 @@ def _is_include_directive(entry): """ return (isinstance(entry, list) and len(entry) == 2 and entry[0] == 'include' and - isinstance(entry[1], six.string_types)) + isinstance(entry[1], str)) def _is_ssl_on_directive(entry): """Checks if an nginx parsed entry is an 'ssl on' directive. @@ -654,7 +655,7 @@ def _add_directive(block, directive, insert_at_top): directive_name = directive[0] def can_append(loc, dir_name): """ Can we append this directive to the block? """ - return loc is None or (isinstance(dir_name, six.string_types) + return loc is None or (isinstance(dir_name, str) and dir_name in REPEATABLE_DIRECTIVES) err_fmt = 'tried to insert directive "{0}" but found conflicting "{1}".' diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 390e18e4d..e55d48dc4 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -4,7 +4,6 @@ raw lists of tokens from pyparsing. """ import abc import logging -import six from acme.magic_typing import List from certbot import errors @@ -14,7 +13,7 @@ COMMENT = " managed by Certbot" COMMENT_BLOCK = ["#", COMMENT] -class Parsable(object): +class Parsable: """ Abstract base class for "Parsable" objects whose underlying representation is a tree of lists. @@ -152,7 +151,7 @@ class Statements(Parsable): if not isinstance(raw_list, list): raise errors.MisconfigurationError("Statements parsing expects a list!") # If there's a trailing whitespace in the list of statements, keep track of it. - if raw_list and isinstance(raw_list[-1], six.string_types) and raw_list[-1].isspace(): + if raw_list and isinstance(raw_list[-1], str) and raw_list[-1].isspace(): self._trailing_whitespace = raw_list[-1] raw_list = raw_list[:-1] self._data = [parse_raw(elem, self, add_spaces) for elem in raw_list] @@ -184,7 +183,7 @@ class Statements(Parsable): def _space_list(list_): """ Inserts whitespace between adjacent non-whitespace tokens. """ spaced_statement = [] # type: List[str] - for i in reversed(six.moves.xrange(len(list_))): + for i in reversed(range(len(list_))): spaced_statement.insert(0, list_[i]) if i > 0 and not list_[i].isspace() and not list_[i-1].isspace(): spaced_statement.insert(0, " ") @@ -206,7 +205,7 @@ class Sentence(Parsable): :returns: whether this lists is parseable by `Sentence`. """ return isinstance(lists, list) and len(lists) > 0 and \ - all(isinstance(elem, six.string_types) for elem in lists) + all(isinstance(elem, str) for elem in lists) def parse(self, raw_list, add_spaces=False): """ Parses a list of string types into this object. @@ -214,7 +213,7 @@ class Sentence(Parsable): if add_spaces: raw_list = _space_list(raw_list) if not isinstance(raw_list, list) or \ - any(not isinstance(elem, six.string_types) for elem in raw_list): + any(not isinstance(elem, str) for elem in raw_list): raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") self._data = raw_list diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index a4faf470e..8b638f28e 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,32 +1,19 @@ -from distutils.version import LooseVersion -import sys - -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'acme>=1.4.0', 'certbot>=1.6.0', - 'PyOpenSSL', - 'pyparsing>=1.5.5', # Python3 support - 'setuptools', + 'PyOpenSSL>=17.3.0', + 'pyparsing>=2.2.0', + 'setuptools>=39.0.1', 'zope.interface', ] -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - setup( name='certbot-nginx', version=version, @@ -35,7 +22,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', @@ -43,12 +30,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index e677ad471..9ccc3fc9e 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -39,7 +39,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(12, len(self.config.parser.parsed)) + self.assertEqual(13, len(self.config.parser.parsed)) @mock.patch("certbot_nginx._internal.configurator.util.exe_exists") @mock.patch("certbot_nginx._internal.configurator.subprocess.Popen") @@ -89,7 +89,7 @@ class NginxConfiguratorTest(util.NginxTest): "155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", - "headers.com", "example.net"}) + "headers.com", "example.net", "ssl.both.com"}) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -842,7 +842,7 @@ class NginxConfiguratorTest(util.NginxTest): self.config.recovery_routine() self.config.revert_challenge_config() self.config.rollback_checkpoints() - self.assertTrue(mock_parser_load.call_count == 3) + self.assertEqual(mock_parser_load.call_count, 3) def test_choose_vhosts_wildcard(self): # pylint: disable=protected-access @@ -935,7 +935,19 @@ class NginxConfiguratorTest(util.NginxTest): prefer_ssl=False, no_ssl_filter_port='80') # Check that the dialog was called with only port 80 vhosts - self.assertEqual(len(mock_select_vhs.call_args[0][0]), 6) + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 8) + + def test_choose_auth_vhosts(self): + """choose_auth_vhosts correctly selects duplicative and HTTP/HTTPS vhosts""" + http, https = self.config.choose_auth_vhosts('ssl.both.com') + self.assertEqual(len(http), 4) + self.assertEqual(len(https), 2) + self.assertEqual(http[0].names, {'ssl.both.com'}) + self.assertEqual(http[1].names, {'ssl.both.com'}) + self.assertEqual(http[2].names, {'ssl.both.com'}) + self.assertEqual(http[3].names, {'*.both.com'}) + self.assertEqual(https[0].names, {'ssl.both.com'}) + self.assertEqual(https[1].names, {'*.both.com'}) class InstallSslOptionsConfTest(util.NginxTest): diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index 8f0673c1f..d0e84fc83 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -6,7 +6,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six from acme import challenges from certbot import achallenges @@ -45,6 +44,10 @@ class HttpPerformTest(util.NginxTest): challb=acme_util.chall_to_challb( challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), domain="migration.com", account_key=account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), + domain="ipv6ssl.com", account_key=account_key), ] def setUp(self): @@ -78,8 +81,8 @@ class HttpPerformTest(util.NginxTest): http_responses = self.http01.perform() - self.assertEqual(len(http_responses), 4) - for i in six.moves.range(4): + self.assertEqual(len(http_responses), 5) + for i in range(5): self.assertEqual(http_responses[i], acme_responses[i]) def test_mod_config(self): @@ -106,6 +109,43 @@ class HttpPerformTest(util.NginxTest): # self.assertEqual(vhost.addrs, set(v_addr2_print)) # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')])) + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_http_and_https(self, mock_add_server_directives): + """A server_name with both HTTP and HTTPS vhosts should get modded in both vhosts""" + self.configuration.https_port = 443 + self.http01.add_chall(self.achalls[3]) # migration.com + self.http01._mod_config() # pylint: disable=protected-access + + # Domain has an HTTP and HTTPS vhost + # 2 * 'rewrite' + 2 * 'return 200 keyauthz' = 4 + self.assertEqual(mock_add_server_directives.call_count, 4) + + @mock.patch('certbot_nginx._internal.parser.nginxparser.dump') + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_only_https(self, mock_add_server_directives, mock_dump): + """A server_name with only an HTTPS vhost should get modded""" + self.http01.add_chall(self.achalls[4]) # ipv6ssl.com + self.http01._mod_config() # pylint: disable=protected-access + + # It should modify the existing HTTPS vhost + self.assertEqual(mock_add_server_directives.call_count, 2) + # since there was no suitable HTTP vhost or default HTTP vhost, a non-empty one + # should have been created and written to the challenge conf file + self.assertNotEqual(mock_dump.call_args[0][0], []) + + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_deduplicate(self, mock_add_server_directives): + """A vhost that appears in both HTTP and HTTPS vhosts only gets modded once""" + achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), + domain="ssl.both.com", account_key=AUTH_KEY) + self.http01.add_chall(achall) + self.http01._mod_config() # pylint: disable=protected-access + + # Should only get called 5 times, rather than 6, because two vhosts are the same + self.assertEqual(mock_add_server_directives.call_count, 5*2) + @mock.patch("certbot_nginx._internal.configurator.NginxConfigurator.ipv6_info") def test_default_listen_addresses_no_memoization(self, ipv6_info): # pylint: disable=protected-access diff --git a/certbot-nginx/tests/obj_test.py b/certbot-nginx/tests/obj_test.py index 93b9493eb..f385649ed 100644 --- a/certbot-nginx/tests/obj_test.py +++ b/certbot-nginx/tests/obj_test.py @@ -75,7 +75,7 @@ class AddrTest(unittest.TestCase): new_addr1 = Addr.fromstring("192.168.1.1 spdy") self.assertEqual(self.addr1, new_addr1) self.assertNotEqual(self.addr1, self.addr2) - self.assertFalse(self.addr1 == 3333) + self.assertNotEqual(self.addr1, 3333) def test_equivalent_any_addresses(self): from certbot_nginx._internal.obj import Addr @@ -168,7 +168,7 @@ class VirtualHostTest(unittest.TestCase): self.assertEqual(vhost1b, self.vhost1) self.assertEqual(str(vhost1b), str(self.vhost1)) - self.assertFalse(vhost1b == 1234) + self.assertNotEqual(vhost1b, 1234) def test_str(self): stringified = '\n'.join(['file: filep', 'addrs: localhost', diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 620d1b6de..23fe390ad 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -51,6 +51,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual({nparser.abs_path(x) for x in ['foo.conf', 'nginx.conf', 'server.conf', 'sites-enabled/default', + 'sites-enabled/both.com', 'sites-enabled/example.com', 'sites-enabled/headers.com', 'sites-enabled/migration.com', @@ -88,7 +89,7 @@ class NginxParserTest(util.NginxTest): parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(9, len( + self.assertEqual(10, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -171,7 +172,7 @@ class NginxParserTest(util.NginxTest): '*.www.example.com'}, [], [2, 1, 0]) - self.assertEqual(14, len(vhosts)) + self.assertEqual(19, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] @@ -492,6 +493,14 @@ class NginxParserTest(util.NginxTest): self.assertEqual(['server'], parsed[0][2][0]) self.assertEqual(['listen', '80'], parsed[0][2][1][3]) + def test_valid_unicode_roundtrip(self): + """This tests the parser's ability to load and save a config containing Unicode""" + nparser = parser.NginxParser(self.config_path) + nparser._parse_files( + nparser.abs_path('valid_unicode_comments.conf') + ) # pylint: disable=protected-access + nparser.filedump(lazy=False) + def test_invalid_unicode_characters(self): with self.assertLogs() as log: nparser = parser.NginxParser(self.config_path) diff --git a/certbot-nginx/tests/test_log_util.py b/certbot-nginx/tests/test_log_util.py deleted file mode 100644 index 7aebf2151..000000000 --- a/certbot-nginx/tests/test_log_util.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Backport for `TestCase.assertLogs()`. - -Most of the idea and code are from CPython implementation. -https://github.com/python/cpython/blob/b76518d43fb82ed9e5d27025d18c90a23d525c90/Lib/unittest/case.py -""" -import logging -import collections - -__all__ = ['AssertLogsMixin'] - -LoggingWatcher = collections.namedtuple('LoggingWatcher', ['records', 'output']) - - -class CapturingHandler(logging.Handler): - """ - A logging handler capturing all (raw and formatted) logging output. - """ - - def __init__(self): - super(CapturingHandler, self).__init__() - self.watcher = LoggingWatcher([], []) - - def flush(self): - pass - - def emit(self, record): - self.watcher.records.append(record) - self.watcher.output.append(self.format(record)) - - - -class AssertLogsContext(object): - """ - A context manager used to implement `TestCase.assertLogs()`. - """ - - LOGGING_FORMAT = '%(levelname)s:%(name)s:%(message)s' - - def __init__(self, test_case, logger_name, level): - self.test_case = test_case - - self.logger_name = logger_name - self.logger_states = None - self.logger = None - - if level: - # pylint: disable=protected-access,no-member - try: - # In Python 3.x - name_to_level = logging._nameToLevel # type: ignore - except AttributeError: - # In Python 2.7 - name_to_level = logging._levelNames # type: ignore - - self.level = name_to_level.get(level, level) - else: - self.level = logging.INFO - - self.watcher = None - - def _save_logger_states(self): - self.logger_states = (self.logger.handlers[:], self.logger.level, self.logger.propagate) - - def _restore_logger_states(self): - self.logger.handlers, self.logger.level, self.logger.propagate = self.logger_states - - def __enter__(self): - if isinstance(self.logger_name, logging.Logger): - self.logger = self.logger_name - else: - self.logger = logging.getLogger(self.logger_name) - - formatter = logging.Formatter(self.LOGGING_FORMAT) - - handler = CapturingHandler() - handler.setFormatter(formatter) - - self._save_logger_states() - self.logger.handlers = [handler] - self.logger.setLevel(self.level) - self.logger.propagate = False - - self.watcher = handler.watcher - return handler.watcher - - def __exit__(self, exc_type, exc_value, tb): - self._restore_logger_states() - - if exc_type is not None: - # let unexpected exceptions pass through - return - - if not self.watcher.records: - self._raiseFailure( - "no logs of level {} or higher triggered on {}" - .format(logging.getLevelName(self.level), self.logger.name)) - - def _raiseFailure(self, message): - message = self.test_case._formatMessage(None, message) # pylint: disable=protected-access - raise self.test_case.failureException(message) - - -class AssertLogsMixin(object): - """ - A mixin that implements `TestCase.assertLogs()`. - """ - - def assertLogs(self, logger=None, level=None): - """Fail unless a log message of level *level* or higher is emitted - on *logger_name* or its children. If omitted, *level* defaults to - INFO and *logger* defaults to the root logger. - This method must be used as a context manager, and will yield - a recording object with two attributes: `output` and `records`. - At the end of the context manager, the `output` attribute will - be a list of the matching formatted log messages and the - `records` attribute will be a list of the corresponding LogRecord - objects. - Example:: - with self.assertLogs('foo', level='INFO') as cm: - logging.getLogger('foo').info('first message') - logging.getLogger('foo.bar').error('second message') - self.assertEqual(cm.output, ['INFO:foo:first message', - 'ERROR:foo.bar:second message']) - """ - return AssertLogsContext(self, logger, level) diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index f545dc5bc..ee53e8e1e 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -17,10 +17,9 @@ from certbot.plugins import common from certbot.tests import util as test_util from certbot_nginx._internal import configurator from certbot_nginx._internal import nginxparser -import test_log_util -class NginxTest(test_log_util.AssertLogsMixin, test_util.ConfigTestCase): +class NginxTest(test_util.ConfigTestCase): def setUp(self): super(NginxTest, self).setUp() diff --git a/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com b/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com new file mode 100644 index 000000000..23a660df3 --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com @@ -0,0 +1,32 @@ +server { + server_name ssl.both.com; +} + +# a duplicate vhost +server { + server_name ssl.both.com; +} + +# a duplicate by means of wildcard +server { + server_name *.both.com; +} + +# combined HTTP and HTTPS +server { + server_name ssl.both.com; + listen 80; + listen 5001 ssl; + + ssl_certificate cert.pem; + ssl_certificate_key cert.key; +} + +# HTTPS, duplicate by means of wildcard +server { + server_name *.both.com; + listen 5001 ssl; + + ssl_certificate cert.pem; + ssl_certificate_key cert.key; +} diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index fc83d3b46..c316ec376 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,24 +2,136 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.10.0 - master +## 1.14.0 - master + +### Added + +* + +### Changed + +* certbot-auto no longer checks for updates on any operating system. + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + +## 1.13.0 - 2021-03-02 + +### Added + +* + +### Changed + +* CLI flags `--os-packages-only`, `--no-self-upgrade`, `--no-bootstrap` and `--no-permissions-check`, + which are related to certbot-auto, are deprecated and will be removed in a future release. +* Certbot no longer conditionally depends on an external mock module. Certbot's + test API will continue to use it if it is available for backwards + compatibility, however, this behavior has been deprecated and will be removed + in a future release. +* The acme library no longer depends on the `security` extras from `requests` + which was needed to support SNI in TLS requests when using old versions of + Python 2. +* Certbot and all of its components no longer depend on the library `six`. +* The update of certbot-auto itself is now disabled on all RHEL-like systems. +* When revoking a certificate by `--cert-name`, it is no longer necessary to specify the `--server` + if the certificate was obtained from a non-default ACME server. +* The nginx authenticator now configures all matching HTTP and HTTPS vhosts for the HTTP-01 + challenge. It is now compatible with external HTTPS redirection by a CDN or load balancer. + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + +## 1.12.0 - 2021-02-02 + +### Added + +* + +### Changed + +* The `--preferred-chain` flag now only checks the Issuer Common Name of the + topmost (closest to the root) certificate in the chain, instead of checking + every certificate in the chain. + See [#8577](https://github.com/certbot/certbot/issues/8577). +* Support for Python 2 has been removed. +* In previous releases, we caused certbot-auto to stop updating its Certbot + installation. In this release, we are beginning to disable updates to the + certbot-auto script itself. This release includes Amazon Linux users, and all + other systems that are not based on Debian or RHEL. We plan to make this + change to the certbot-auto script for all users in the coming months. + +### Fixed + +* Fixed the apache component on openSUSE Tumbleweed which no longer provides + an apache2ctl symlink and uses apachectl instead. +* Fixed a typo in `certbot/crypto_util.py` causing an error upon attempting `secp521r1` key generation + +More details about these changes can be found on our GitHub repo. + +## 1.11.0 - 2021-01-05 + +### Added + +* + +### Changed + +* We deprecated support for Python 2 in Certbot and its ACME library. + Support for Python 2 will be removed in the next planned release of Certbot. +* certbot-auto was deprecated on all systems. For more information about this + change, see + https://community.letsencrypt.org/t/certbot-auto-no-longer-works-on-debian-based-systems/139702/7. +* We deprecated support for Apache 2.2 in the certbot-apache plugin and it will + be removed in a future release of Certbot. + +### Fixed + +* The Certbot snap no longer loads packages installed via `pip install --user`. This + was unintended and DNS plugins should be installed via `snap` instead. +* `certbot-dns-google` would sometimes crash with HTTP 409/412 errors when used with very large zones. See [#6036](https://github.com/certbot/certbot/issues/6036). +* `certbot-dns-google` would sometimes crash with an HTTP 412 error if preexisting records had an unexpected TTL, i.e.: different than Certbot's default TTL for this plugin. See [#8551](https://github.com/certbot/certbot/issues/8551). + +More details about these changes can be found on our GitHub repo. + +## 1.10.1 - 2020-12-03 + +### Fixed + +* Fixed a bug in `certbot.util.add_deprecated_argument` that caused the + deprecated `--manual-public-ip-logging-ok` flag to crash Certbot in some + scenarios. + +More details about these changes can be found on our GitHub repo. + +## 1.10.0 - 2020-12-01 ### Added * Added timeout to DNS query function calls for dns-rfc2136 plugin. * Confirmation when deleting certificates -* +* CLI flag `--key-type` has been added to specify 'rsa' or 'ecdsa' (default 'rsa'). +* CLI flag `--elliptic-curve` has been added which takes an NIST/SECG elliptic curve. Any of + `secp256r1`, `secp384r1` and `secp521r1` are accepted values. +* The command `certbot certficates` lists the which type of the private key that was used + for the private key. +* Support for Python 3.9 was added to Certbot and all of its components. ### Changed * certbot-auto was deprecated on Debian based systems. * CLI flag `--manual-public-ip-logging-ok` is now a no-op, generates a deprecation warning, and will be removed in a future release. -* ### Fixed -* +* Fixed a Unicode-related crash in the nginx plugin when running under Python 2. More details about these changes can be found on our GitHub repo. @@ -55,7 +167,7 @@ More details about these changes can be found on our GitHub repo. ### Added -* Added the ability to remove email and phone contact information from an account +* Added the ability to remove email and phone contact information from an account using `update_account --register-unsafely-without-email` ### Changed @@ -67,7 +179,7 @@ More details about these changes can be found on our GitHub repo. * The problem causing the Apache plugin in the Certbot snap on ARM systems to fail to load the Augeas library it depends on has been fixed. * The `acme` library can now tell the ACME server to clear contact information by passing an empty - `tuple` to the `contact` field of a `Registration` message. + `tuple` to the `contact` field of a `Registration` message. * Fixed the `*** stack smashing detected ***` error in the Certbot snap on some systems. More details about these changes can be found on our GitHub repo. diff --git a/certbot/README.rst b/certbot/README.rst index 6d04e6fa3..40f6a52ec 100644 --- a/certbot/README.rst +++ b/certbot/README.rst @@ -18,10 +18,6 @@ systems. To see the changes made to Certbot between versions please refer to our `changelog `_. -Until May 2016, Certbot was named simply ``letsencrypt`` or ``letsencrypt-auto``, -depending on install method. Instructions on the Internet, and some pieces of the -software, may still refer to this older name. - Contributing ------------ @@ -96,7 +92,7 @@ Current Features - apache/2.x - nginx/0.8.48+ - webroot (adds files to webroot directories in order to prove control of - domains and obtain certs) + domains and obtain certificates) - standalone (runs its own simple webserver to prove you control a domain) - other server software via `third party plugins `_ @@ -106,6 +102,8 @@ Current Features * Can get domain-validated (DV) certificates. * Can revoke certificates. * Adjustable RSA key bit-length (2048 (default), 4096, ...). +* Adjustable `EC `_ + key (`secp256r1` (default), `secp384r1`, `secp521r1`). * Can optionally install a http -> https redirect, so your site effectively runs https only (Apache only) * Fully automated. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 8ee415682..790443bcc 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,4 +1,3 @@ """Certbot client.""" - # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.10.0.dev0' +__version__ = '1.14.0.dev0' diff --git a/certbot/certbot/_internal/account.py b/certbot/certbot/_internal/account.py index 8cfe5ea11..b2d50297e 100644 --- a/certbot/certbot/_internal/account.py +++ b/certbot/certbot/_internal/account.py @@ -10,7 +10,6 @@ from cryptography.hazmat.primitives import serialization import josepy as jose import pyrfc3339 import pytz -import six from acme import fields as acme_fields from acme import messages @@ -20,11 +19,12 @@ from certbot import interfaces from certbot import util from certbot._internal import constants from certbot.compat import os +from certbot.compat import filesystem logger = logging.getLogger(__name__) -class Account(object): +class Account: """ACME protocol registration. :ivar .RegistrationResource regr: Registration Resource @@ -100,7 +100,7 @@ class AccountMemoryStorage(interfaces.AccountStorage): self.accounts = initial_accounts if initial_accounts is not None else {} def find_all(self): - return list(six.itervalues(self.accounts)) + return list(self.accounts.values()) def save(self, account, client): if account.id in self.accounts: @@ -324,7 +324,7 @@ class AccountFileStorage(interfaces.AccountStorage): if server_path in reused_servers: next_server_path = reused_servers[server_path] next_dir_path = link_func(next_server_path) - if os.path.islink(next_dir_path) and os.readlink(next_dir_path) == dir_path: + if os.path.islink(next_dir_path) and filesystem.readlink(next_dir_path) == dir_path: possible_next_link = True server_path = next_server_path dir_path = next_dir_path @@ -332,7 +332,7 @@ class AccountFileStorage(interfaces.AccountStorage): # if there's not a next one up to delete, then delete me # and whatever I link to while os.path.islink(dir_path): - target = os.readlink(dir_path) + target = filesystem.readlink(dir_path) os.unlink(dir_path) dir_path = target diff --git a/certbot/certbot/_internal/auth_handler.py b/certbot/certbot/_internal/auth_handler.py index 7ea2a1de8..17bf75225 100644 --- a/certbot/certbot/_internal/auth_handler.py +++ b/certbot/certbot/_internal/auth_handler.py @@ -19,7 +19,7 @@ from certbot._internal import error_handler logger = logging.getLogger(__name__) -class AuthHandler(object): +class AuthHandler: """ACME Authorization Handler for a client. :ivar auth: Authenticator capable of solving diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index 5c1c515c2..ee2bd6254 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -65,6 +65,7 @@ def rename_lineage(config): disp.notification("Successfully renamed {0} to {1}." .format(certname, new_certname), pause=False) + def certificates(config): """Display information about certs configured with Certbot @@ -87,6 +88,7 @@ def certificates(config): # Describe all the certs _describe_certs(config, parsed_certs, parse_failures) + def delete(config): """Delete Certbot files associated with a certificate lineage.""" certnames = get_certnames(config, "delete", allow_multiple=True) @@ -123,11 +125,13 @@ def lineage_for_certname(cli_config, certname): logger.debug("Traceback was:\n%s", traceback.format_exc()) return None + def domains_for_certname(config, certname): """Find the domains in the cert with name certname.""" lineage = lineage_for_certname(config, certname) return lineage.names() if lineage else None + def find_duplicative_certs(config, domains): """Find existing certs that match the given domain names. @@ -172,6 +176,7 @@ def find_duplicative_certs(config, domains): return _search_lineages(config, update_certs_for_domain_matches, (None, None)) + def _archive_files(candidate_lineage, filetype): """ In order to match things like: /etc/letsencrypt/archive/example.com/chain1.pem. @@ -193,6 +198,7 @@ def _archive_files(candidate_lineage, filetype): return pattern return None + def _acceptable_matches(): """ Generates the list that's passed to match_and_check_overlaps. Is its own function to make unit testing easier. @@ -203,6 +209,7 @@ def _acceptable_matches(): return [lambda x: x.fullchain_path, lambda x: x.cert_path, lambda x: _archive_files(x, "cert"), lambda x: _archive_files(x, "fullchain")] + def cert_path_to_lineage(cli_config): """ If config.cert_path is defined, try to find an appropriate value for config.certname. @@ -216,9 +223,10 @@ def cert_path_to_lineage(cli_config): """ acceptable_matches = _acceptable_matches() match = match_and_check_overlaps(cli_config, acceptable_matches, - lambda x: cli_config.cert_path[0], lambda x: x.lineagename) + lambda x: cli_config.cert_path, lambda x: x.lineagename) return match[0] + def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func): """ Searches through all lineages for a match, and checks for duplicates. If a duplicate is found, an error is raised, as performing operations on lineages @@ -246,7 +254,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func matched = _search_lineages(cli_config, find_matches, [], acceptable_matches) if not matched: - raise errors.Error("No match found for cert-path {0}!".format(cli_config.cert_path[0])) + raise errors.Error("No match found for cert-path {0}!".format(cli_config.cert_path)) elif len(matched) > 1: raise errors.OverlappingMatchFound() return matched @@ -284,20 +292,23 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False): valid_string = "{0} ({1})".format(cert.target_expiry, status) serial = format(crypto_util.get_serial_from_cert(cert.cert_path), 'x') - certinfo.append(" Certificate Name: {0}\n" - " Serial Number: {1}\n" - " Domains: {2}\n" - " Expiry Date: {3}\n" - " Certificate Path: {4}\n" - " Private Key Path: {5}".format( + certinfo.append(" Certificate Name: {}\n" + " Serial Number: {}\n" + " Key Type: {}\n" + " Domains: {}\n" + " Expiry Date: {}\n" + " Certificate Path: {}\n" + " Private Key Path: {}".format( cert.lineagename, serial, + cert.private_key_type, " ".join(cert.names()), valid_string, cert.fullchain, cert.privkey)) return "".join(certinfo) + def get_certnames(config, verb, allow_multiple=False, custom_prompt=None): """Get certname from flag, interactively, or error out. """ @@ -337,10 +348,12 @@ def get_certnames(config, verb, allow_multiple=False, custom_prompt=None): # Private Helpers ################### + def _report_lines(msgs): """Format a results report for a category of single-line renewal outcomes""" return " " + "\n ".join(str(msg) for msg in msgs) + def _report_human_readable(config, parsed_certs): """Format a results report for a parsed cert""" certinfo = [] @@ -348,6 +361,7 @@ def _report_human_readable(config, parsed_certs): certinfo.append(human_readable_cert_info(config, cert)) return "\n".join(certinfo) + def _describe_certs(config, parsed_certs, parse_failures): """Print information about the certs we know about""" out = [] # type: List[str] @@ -355,7 +369,7 @@ def _describe_certs(config, parsed_certs, parse_failures): notify = out.append if not parsed_certs and not parse_failures: - notify("No certs found.") + notify("No certificates found.") else: if parsed_certs: match = "matching " if config.certname or config.domains else "" @@ -369,6 +383,7 @@ def _describe_certs(config, parsed_certs, parse_failures): disp = zope.component.getUtility(interfaces.IDisplay) disp.notification("\n".join(out), pause=False, wrap=False) + def _search_lineages(cli_config, func, initial_rv, *args): """Iterate func over unbroken lineages, allowing custom return conditions. diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index f6f648aa9..688efeb7a 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -1,6 +1,5 @@ """Certbot command line argument & config processing.""" # pylint: disable=too-many-lines -from __future__ import print_function import logging import logging.handlers import argparse @@ -28,7 +27,8 @@ from certbot._internal.cli.cli_constants import ( ARGPARSE_PARAMS_TO_REMOVE, EXIT_ACTIONS, ZERO_ARG_ACTIONS, - VAR_MODIFIERS + VAR_MODIFIERS, + DEPRECATED_OPTIONS ) from certbot._internal.cli.cli_utils import ( @@ -248,27 +248,6 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): default=flag_default("duplicate"), help="Allow making a certificate lineage that duplicates an existing one " "(both can be renewed in parallel)") - helpful.add( - "automation", "--os-packages-only", action="store_true", - default=flag_default("os_packages_only"), - help="(certbot-auto only) install OS package dependencies and then stop") - helpful.add( - "automation", "--no-self-upgrade", action="store_true", - default=flag_default("no_self_upgrade"), - help="(certbot-auto only) prevent the certbot-auto script from" - " upgrading itself to newer released versions (default: Upgrade" - " automatically)") - helpful.add( - "automation", "--no-bootstrap", action="store_true", - default=flag_default("no_bootstrap"), - help="(certbot-auto only) prevent the certbot-auto script from" - " installing OS-level dependencies (default: Prompt to install " - " OS-wide dependencies, but exit if the user says 'No')") - helpful.add( - "automation", "--no-permissions-check", action="store_true", - default=flag_default("no_permissions_check"), - help="(certbot-auto only) skip the check on the file system" - " permissions of the certbot-auto script") helpful.add( ["automation", "renew", "certonly", "run"], "-q", "--quiet", dest="quiet", action="store_true", @@ -313,6 +292,16 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): helpful.add( "security", "--rsa-key-size", type=int, metavar="N", default=flag_default("rsa_key_size"), help=config_help("rsa_key_size")) + helpful.add( + "security", "--key-type", choices=['rsa', 'ecdsa'], type=str, + default=flag_default("key_type"), help=config_help("key_type")) + helpful.add( + "security", "--elliptic-curve", type=str, choices=[ + 'secp256r1', + 'secp384r1', + 'secp521r1', + ], metavar="N", + default=flag_default("elliptic_curve"), help=config_help("elliptic_curve")) helpful.add( "security", "--must-staple", action="store_true", dest="must_staple", default=flag_default("must_staple"), @@ -440,6 +429,12 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): default=flag_default("autorenew"), dest="autorenew", help="Disable auto renewal of certificates.") + # Deprecated arguments + helpful.add_deprecated_argument("--os-packages-only", 0) + helpful.add_deprecated_argument("--no-self-upgrade", 0) + helpful.add_deprecated_argument("--no-bootstrap", 0) + helpful.add_deprecated_argument("--no-permissions-check", 0) + # Populate the command line parameters for new style enhancements enhancements.populate_cli(helpful.add) @@ -461,6 +456,11 @@ def set_by_cli(var): (CLI or config file) including if the user explicitly set it to the default. Returns False if the variable was assigned a default value. """ + # We should probably never actually hit this code. But if we do, + # a deprecated option has logically never been set by the CLI. + if var in DEPRECATED_OPTIONS: + return False + detector = set_by_cli.detector # type: ignore if detector is None and helpful_parser is not None: # Setup on first run: `detector` is a weird version of config in which @@ -521,6 +521,9 @@ def option_was_set(option, value): :rtype: bool """ + # If an option is deprecated, it was effectively not set by the user. + if option in DEPRECATED_OPTIONS: + return False return set_by_cli(option) or not has_default_value(option, value) diff --git a/certbot/certbot/_internal/cli/cli_constants.py b/certbot/certbot/_internal/cli/cli_constants.py index 4bc84bfe7..bd25f9bee 100644 --- a/certbot/certbot/_internal/cli/cli_constants.py +++ b/certbot/certbot/_internal/cli/cli_constants.py @@ -105,3 +105,14 @@ VAR_MODIFIERS = {"account": {"server",}, "renew_hook": {"deploy_hook",}, "server": {"dry_run", "staging",}, "webroot_map": {"webroot_path",}} + +# This is a list of all CLI options that we have ever deprecated. It lets us +# opt out of the default detection, which can interact strangely with option +# deprecation. See https://github.com/certbot/certbot/issues/8540 for more info. +DEPRECATED_OPTIONS = { + "manual_public_ip_logging_ok", + "os_packages_only", + "no_self_upgrade", + "no_bootstrap", + "no_permissions_check", +} diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index a0ddce38f..5bdbbe02c 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -12,7 +12,7 @@ from certbot.compat import os from certbot._internal import constants -class _Default(object): +class _Default: """A class to use as a default to detect if a value is set by a user""" def __bool__(self): @@ -66,7 +66,7 @@ def config_help(name, hidden=False): return field.__doc__ -class HelpfulArgumentGroup(object): +class HelpfulArgumentGroup: """Emulates an argparse group for use with HelpfulArgumentParser. This class is used in the add_group method of HelpfulArgumentParser. diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index a86ff3743..2505c24c6 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -1,11 +1,12 @@ """Certbot command line argument parser""" -from __future__ import print_function + import argparse import copy +import functools import glob import sys + import configargparse -import six import zope.component import zope.interface @@ -40,7 +41,7 @@ from certbot._internal.cli import ( ) -class HelpfulArgumentParser(object): +class HelpfulArgumentParser: """Argparse Wrapper. This class wraps argparse, adding the ability to make --help less @@ -97,7 +98,7 @@ class HelpfulArgumentParser(object): if isinstance(help1, bool) and isinstance(help2, bool): self.help_arg = help1 or help2 else: - self.help_arg = help1 if isinstance(help1, six.string_types) else help2 + self.help_arg = help1 if isinstance(help1, str) else help2 short_usage = self._usage_string(plugins, self.help_arg) @@ -230,6 +231,10 @@ class HelpfulArgumentParser(object): raise errors.Error( "Parameters --hsts and --auto-hsts cannot be used simultaneously.") + if isinstance(parsed_args.key_type, list) and len(parsed_args.key_type) > 1: + raise errors.Error( + "Only *one* --key-type type may be provided at this time.") + return parsed_args def set_test_server(self, parsed_args): @@ -352,6 +357,18 @@ class HelpfulArgumentParser(object): :param dict **kwargs: various argparse settings for this argument """ + action = kwargs.get("action") + if action is util.DeprecatedArgumentAction: + # If the argument is deprecated through + # certbot.util.add_deprecated_argument, it is not shown in the help + # output and any value given to the argument is thrown away during + # argument parsing. Because of this, we handle this case early + # skipping putting the argument in different help topics and + # handling default detection since these actions aren't needed and + # can cause bugs like + # https://github.com/certbot/certbot/issues/8495. + self.parser.add_argument(*args, **kwargs) + return if isinstance(topics, list): # if this flag can be listed in multiple sections, try to pick the one @@ -406,8 +423,22 @@ class HelpfulArgumentParser(object): :param int nargs: Number of arguments the option takes. """ - util.add_deprecated_argument( - self.parser.add_argument, argument_name, num_args) + # certbot.util.add_deprecated_argument expects the normal add_argument + # interface provided by argparse. This is what is given including when + # certbot.util.add_deprecated_argument is used by plugins, however, in + # that case the first argument to certbot.util.add_deprecated_argument + # is certbot._internal.cli.HelpfulArgumentGroup.add_argument which + # internally calls the add method of this class. + # + # The difference between the add method of this class and the standard + # argparse add_argument method caused a bug in the past (see + # https://github.com/certbot/certbot/issues/8495) so we use the same + # code path here for consistency and to ensure it works. To do that, we + # wrap the add method in a similar way to + # HelpfulArgumentGroup.add_argument by providing a help topic (which in + # this case is set to None). + add_func = functools.partial(self.add, None) + util.add_deprecated_argument(add_func, argument_name, num_args) def add_group(self, topic, verbs=(), **kwargs): """Create a new argument group. @@ -438,7 +469,7 @@ class HelpfulArgumentParser(object): may or may not be displayed as help topics. """ - for name, plugin_ep in six.iteritems(plugins): + for name, plugin_ep in plugins.items(): parser_or_group = self.add_group(name, description=plugin_ep.long_description) plugin_ep.plugin_cls.inject_parser_options(parser_or_group, name) diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py index 62f5e224d..43ee41c88 100644 --- a/certbot/certbot/_internal/cli/paths_parser.py +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -2,7 +2,6 @@ paths for certificates""" from certbot.compat import os from certbot._internal.cli import ( - read_file, flag_default, config_help ) @@ -14,22 +13,19 @@ def _paths_parser(helpful): if verb == "help": verb = helpful.help_arg - cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked." - sections = ["paths", "install", "revoke", "certonly", "manage"] + cpkwargs = { + "type": os.path.abspath, + "help": "Path to where certificate is saved (with certonly --csr), installed " + "from, or revoked" + } if verb == "certonly": - add(sections, "--cert-path", type=os.path.abspath, - default=flag_default("auth_cert_path"), help=cph) - elif verb == "revoke": - add(sections, "--cert-path", type=read_file, required=False, help=cph) - else: - add(sections, "--cert-path", type=os.path.abspath, help=cph) + cpkwargs["default"] = flag_default("auth_cert_path") + add(["paths", "install", "revoke", "certonly", "manage"], "--cert-path", **cpkwargs) section = "paths" if verb in ("install", "revoke"): section = verb - # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", - type=((verb == "revoke" and read_file) or os.path.abspath), + add(section, "--key-path", type=os.path.abspath, help="Path to private key for certificate installation " "or revocation (if account key is missing)") diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index b198a8c27..f2fc06937 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -93,7 +93,7 @@ def ua_flags(config): flags.append("hook") return " ".join(flags) -class DummyConfig(object): +class DummyConfig: "Shim for computing a sample user agent." def __init__(self): self.authenticator = "XXX" @@ -227,7 +227,7 @@ def perform_registration(acme, config, tos_cb): raise -class Client(object): +class Client: """Certbot's client. :ivar .IConfig config: Client configuration. @@ -312,7 +312,6 @@ class Client(object): :rtype: tuple """ - # We need to determine the key path, key PEM data, CSR path, # and CSR PEM data. For a dry run, the paths are None because # they aren't permanently saved to disk. For a lineage with @@ -335,16 +334,41 @@ class Client(object): # The key is set to None here but will be created below. key = None + key_size = self.config.rsa_key_size + elliptic_curve = None + + # key-type defaults to a list, but we are only handling 1 currently + if isinstance(self.config.key_type, list): + self.config.key_type = self.config.key_type[0] + if self.config.elliptic_curve and self.config.key_type == 'ecdsa': + elliptic_curve = self.config.elliptic_curve + self.config.auth_chain_path = "./chain-ecdsa.pem" + self.config.auth_cert_path = "./cert-ecdsa.pem" + self.config.key_path = "./key-ecdsa.pem" + elif self.config.rsa_key_size and self.config.key_type.lower() == 'rsa': + key_size = self.config.rsa_key_size + # Create CSR from names if self.config.dry_run: - key = key or util.Key(file=None, - pem=crypto_util.make_key(self.config.rsa_key_size)) + key = key or util.Key( + file=None, + pem=crypto_util.make_key( + bits=key_size, + elliptic_curve=elliptic_curve, + key_type=self.config.key_type, + + ), + ) csr = util.CSR(file=None, form="pem", data=acme_crypto_util.make_csr( key.pem, domains, self.config.must_staple)) else: - key = key or crypto_util.init_save_key(self.config.rsa_key_size, - self.config.key_dir) + key = key or crypto_util.init_save_key( + key_size=key_size, + key_dir=self.config.key_dir, + key_type=self.config.key_type, + elliptic_curve=elliptic_curve, + ) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) orderr = self._get_order_and_authorizations(csr.data, self.config.allow_subset_of_names) diff --git a/certbot/certbot/_internal/configuration.py b/certbot/certbot/_internal/configuration.py index f1e85f9fe..aee0022b8 100644 --- a/certbot/certbot/_internal/configuration.py +++ b/certbot/certbot/_internal/configuration.py @@ -1,7 +1,7 @@ """Certbot user-supplied configuration.""" import copy +from urllib import parse -from six.moves.urllib import parse import zope.interface from certbot import errors @@ -13,7 +13,7 @@ from certbot.compat import os @zope.interface.implementer(interfaces.IConfig) -class NamespaceConfig(object): +class NamespaceConfig: """Configuration wrapper around :class:`argparse.Namespace`. For more documentation, including available attributes, please see diff --git a/certbot/certbot/_internal/constants.py b/certbot/certbot/_internal/constants.py index 9e9373f13..f79d6b9db 100644 --- a/certbot/certbot/_internal/constants.py +++ b/certbot/certbot/_internal/constants.py @@ -57,6 +57,8 @@ CLI_DEFAULTS = dict( https_port=443, break_my_certs=False, rsa_key_size=2048, + elliptic_curve="secp256r1", + key_type="rsa", must_staple=False, redirect=None, auto_hsts=False, diff --git a/certbot/certbot/_internal/display/completer.py b/certbot/certbot/_internal/display/completer.py index 03719862b..a6c984195 100644 --- a/certbot/certbot/_internal/display/completer.py +++ b/certbot/certbot/_internal/display/completer.py @@ -8,7 +8,7 @@ except ImportError: import certbot._internal.display.dummy_readline as readline # type: ignore -class Completer(object): +class Completer: """Provides Tab completion when prompting users for a path. This class is meant to be used with readline to provide Tab diff --git a/certbot/certbot/_internal/error_handler.py b/certbot/certbot/_internal/error_handler.py index 60fb287a6..05af9d837 100644 --- a/certbot/certbot/_internal/error_handler.py +++ b/certbot/certbot/_internal/error_handler.py @@ -45,7 +45,7 @@ else: _SIGNALS = [] -class ErrorHandler(object): +class ErrorHandler: """Context manager for running code that must be cleaned up on failure. The context manager allows you to register functions that will be called diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index 5526b21c4..256fce532 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -1,5 +1,4 @@ """Facilities for implementing hooks that call shell commands.""" -from __future__ import print_function import logging diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index aa0b80eaa..32142fe44 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -38,7 +38,7 @@ def lock_dir(dir_path): return LockFile(os.path.join(dir_path, '.certbot.lock')) -class LockFile(object): +class LockFile: """ Platform independent file lock system. LockFile accepts a parameter, the path to a file acting as a lock. Once the LockFile, @@ -97,7 +97,7 @@ class LockFile(object): return self._lock_mechanism.is_locked() -class _BaseLockMechanism(object): +class _BaseLockMechanism: def __init__(self, path): # type: (str) -> None """ diff --git a/certbot/certbot/_internal/log.py b/certbot/certbot/_internal/log.py index 90dc2cda1..d8a55e27d 100644 --- a/certbot/certbot/_internal/log.py +++ b/certbot/certbot/_internal/log.py @@ -19,7 +19,7 @@ The preferred method to display important information to the user is to use `certbot.display.util` and `certbot.display.ops`. """ -from __future__ import print_function + import functools import logging diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index e02737c4c..9b2141b5a 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -1,6 +1,5 @@ """Certbot main entry point.""" # pylint: disable=too-many-lines -from __future__ import print_function import functools import logging.handlers @@ -11,7 +10,7 @@ import josepy as jose import zope.component from acme import errors as acme_errors -from acme.magic_typing import Union, Iterable, Optional # pylint: disable=unused-import +from acme.magic_typing import Union, Iterable, Optional, List, Tuple # pylint: disable=unused-import import certbot from certbot import crypto_util from certbot import errors @@ -113,12 +112,24 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N if lineage is not None: # Renewal, where we already know the specific lineage we're # interested in - logger.info("Renewing an existing certificate") + display_util.notify( + "{action} for {domains}".format( + action="Simulating renewal of an existing certificate" + if config.dry_run else "Renewing an existing certificate", + domains=display_util.summarize_domain_list(domains or lineage.names()) + ) + ) renewal.renew_cert(config, domains, le_client, lineage) else: # TREAT AS NEW REQUEST assert domains is not None - logger.info("Obtaining a new certificate") + display_util.notify( + "{action} for {domains}".format( + action="Simulating a certificate request" if config.dry_run else + "Requesting a certificate", + domains=display_util.summarize_domain_list(domains) + ) + ) lineage = le_client.obtain_and_enroll_certificate(domains, certname) if lineage is False: raise errors.Error("Certificate could not be obtained") @@ -130,7 +141,33 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N return lineage -def _handle_subset_cert_request(config, domains, cert): +def _handle_unexpected_key_type_migration(config, cert): + # type: (configuration.NamespaceConfig, storage.RenewableCert) -> None + """ + This function ensures that the user will not implicitly migrate an existing key + from one type to another in the situation where a certificate for that lineage + already exist and they have not provided explicitly --key-type and --cert-name. + :param config: Current configuration provided by the client + :param cert: Matching certificate that could be renewed + """ + if not cli.set_by_cli("key_type") or not cli.set_by_cli("certname"): + + new_key_type = config.key_type.upper() + cur_key_type = cert.private_key_type.upper() + + if new_key_type != cur_key_type: + msg = ('Are you trying to change the key type of the certificate named {0} ' + 'from {1} to {2}? Please provide both --cert-name and --key-type on ' + 'the command line confirm the change you are trying to make.') + msg = msg.format(cert.lineagename, cur_key_type, new_key_type) + raise errors.Error(msg) + + +def _handle_subset_cert_request(config, # type: configuration.NamespaceConfig + domains, # type: List[str] + cert # type: storage.RenewableCert + ): + # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] """Figure out what to do if a previous cert had a subset of the names now requested :param config: Configuration object @@ -147,6 +184,8 @@ def _handle_subset_cert_request(config, domains, cert): :rtype: `tuple` of `str` """ + _handle_unexpected_key_type_migration(config, cert) + existing = ", ".join(cert.names()) question = ( "You have an existing certificate that contains a portion of " @@ -175,7 +214,10 @@ def _handle_subset_cert_request(config, domains, cert): raise errors.Error(USER_CANCELLED) -def _handle_identical_cert_request(config, lineage): +def _handle_identical_cert_request(config, # type: configuration.NamespaceConfig + lineage, # type: storage.RenewableCert + ): + # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] """Figure out what to do if a lineage has the same names as a previously obtained one :param config: Configuration object @@ -189,6 +231,8 @@ def _handle_identical_cert_request(config, lineage): :rtype: `tuple` of `str` """ + _handle_unexpected_key_type_migration(config, lineage) + if not lineage.ensure_deployed(): return "reinstall", lineage if renewal.should_renew(config, lineage): @@ -208,7 +252,7 @@ def _handle_identical_cert_request(config, lineage): elif config.verb == "certonly": keep_opt = "Keep the existing certificate for now" choices = [keep_opt, - "Renew & replace the cert (may be subject to CA rate limits)"] + "Renew & replace the certificate (may be subject to CA rate limits)"] display = zope.component.getUtility(interfaces.IDisplay) response = display.menu(question, choices, @@ -264,6 +308,7 @@ def _find_lineage_for_domains(config, domains): return _handle_subset_cert_request(config, domains, subset_names_cert) return None, None + def _find_cert(config, domains, certname): """Finds an existing certificate object given domains and/or a certificate name. @@ -287,7 +332,12 @@ def _find_cert(config, domains, certname): logger.info("Keeping the existing certificate") return (action != "reinstall"), lineage -def _find_lineage_for_domains_and_certname(config, domains, certname): + +def _find_lineage_for_domains_and_certname(config, # type: configuration.NamespaceConfig + domains, # type: List[str] + certname # type: str + ): + # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] """Find appropriate lineage based on given domains and/or certname. :param config: Configuration object @@ -314,8 +364,9 @@ def _find_lineage_for_domains_and_certname(config, domains, certname): if lineage: if domains: if set(cert_manager.domains_for_certname(config, certname)) != set(domains): + _handle_unexpected_key_type_migration(config, lineage) _ask_user_to_confirm_new_names(config, domains, certname, - lineage.names()) # raises if no + lineage.names()) # raises if no return "renew", lineage # unnecessarily specified domains or no domains specified return _handle_identical_cert_request(config, lineage) @@ -381,8 +432,9 @@ def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): _format_list("-", removed), br=os.linesep)) obj = zope.component.getUtility(interfaces.IDisplay) - if not obj.yesno(msg, "Update cert", "Cancel", default=True): - raise errors.ConfigurationError("Specified mismatched cert name and domains.") + if not obj.yesno(msg, "Update certificate", "Cancel", default=True): + raise errors.ConfigurationError("Specified mismatched certificate name and domains.") + def _find_domains_or_certname(config, installer, question=None): """Retrieve domains and certname from config or user input. @@ -459,7 +511,7 @@ def _report_new_cert(config, cert_path, fullchain_path, key_path=None): # and say something more informative here. msg = ('Congratulations! Your certificate and chain have been saved at:{br}' '{0}{br}{1}' - 'Your cert will expire on {2}. To obtain a new or tweaked version of this ' + 'Your certificate will expire on {2}. To obtain a new or tweaked version of this ' 'certificate in the future, simply run {3} again{4}. ' 'To non-interactively renew *all* of your certificates, run "{3} renew"' .format(fullchain_path, privkey_statement, expiry, cli.cli_command, verbswitch, @@ -543,8 +595,8 @@ def _delete_if_appropriate(config): attempt_deletion = config.delete_after_revoke if attempt_deletion is None: - msg = ("Would you like to delete the cert(s) you just revoked, along with all earlier and " - "later versions of the cert?") + msg = ("Would you like to delete the certificate(s) you just revoked, " + "along with all earlier and later versions of the certificate?") attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", force_interactive=True, default=True) @@ -566,8 +618,8 @@ def _delete_if_appropriate(config): cert_manager.match_and_check_overlaps(config, [lambda x: archive_dir], lambda x: x.archive_dir, lambda x: x) except errors.OverlappingMatchFound: - logger.warning("Not deleting revoked certs due to overlapping archive dirs. More than " - "one certificate is using %s", archive_dir) + logger.warning("Not deleting revoked certificates due to overlapping archive dirs. " + "More than one certificate is using %s", archive_dir) return except Exception as e: msg = ('config.default_archive_dir: {0}, config.live_dir: {1}, archive_dir: {2},' @@ -612,7 +664,7 @@ def unregister(config, unused_plugins): :type config: interfaces.IConfig :param unused_plugins: List of plugins (deprecated) - :type unused_plugins: `list` of `str` + :type unused_plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -652,7 +704,7 @@ def register(config, unused_plugins): :type config: interfaces.IConfig :param unused_plugins: List of plugins (deprecated) - :type unused_plugins: `list` of `str` + :type unused_plugins: plugins_disco.PluginsRegistry :returns: `None` or a string indicating and error :rtype: None or str @@ -682,7 +734,7 @@ def update_account(config, unused_plugins): :type config: interfaces.IConfig :param unused_plugins: List of plugins (deprecated) - :type unused_plugins: `list` of `str` + :type unused_plugins: plugins_disco.PluginsRegistry :returns: `None` or a string indicating and error :rtype: None or str @@ -715,7 +767,7 @@ def update_account(config, unused_plugins): acc.regr = acc.regr.update(uri=prev_regr_uri) account_storage.update_regr(acc, cb_client.acme) - if config.email is None: + if not config.email: display_util.notify("Any contact information associated " "with this account has been removed.") else: @@ -759,7 +811,7 @@ def install(config, plugins): :type config: interfaces.IConfig :param plugins: List of plugins - :type plugins: `list` of `str` + :type plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -842,7 +894,7 @@ def plugins_cmd(config, plugins): :type config: interfaces.IConfig :param plugins: List of plugins - :type plugins: `list` of `str` + :type plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -881,7 +933,7 @@ def enhance(config, plugins): :type config: interfaces.IConfig :param plugins: List of plugins - :type plugins: `list` of `str` + :type plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -940,7 +992,7 @@ def rollback(config, plugins): :type config: interfaces.IConfig :param plugins: List of plugins - :type plugins: `list` of `str` + :type plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -958,7 +1010,7 @@ def update_symlinks(config, unused_plugins): :type config: interfaces.IConfig :param unused_plugins: List of plugins (deprecated) - :type unused_plugins: `list` of `str` + :type unused_plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -976,7 +1028,7 @@ def rename(config, unused_plugins): :type config: interfaces.IConfig :param unused_plugins: List of plugins (deprecated) - :type unused_plugins: `list` of `str` + :type unused_plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -994,7 +1046,7 @@ def delete(config, unused_plugins): :type config: interfaces.IConfig :param unused_plugins: List of plugins (deprecated) - :type unused_plugins: `list` of `str` + :type unused_plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -1002,6 +1054,7 @@ def delete(config, unused_plugins): """ cert_manager.delete(config) + def certificates(config, unused_plugins): """Display information about certs configured with Certbot @@ -1009,7 +1062,7 @@ def certificates(config, unused_plugins): :type config: interfaces.IConfig :param unused_plugins: List of plugins (deprecated) - :type unused_plugins: `list` of `str` + :type unused_plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -1017,15 +1070,15 @@ def certificates(config, unused_plugins): """ cert_manager.certificates(config) -# TODO: coop with renewal config -def revoke(config, unused_plugins): + +def revoke(config, unused_plugins: plugins_disco.PluginsRegistry) -> Optional[str]: """Revoke a previously obtained certificate. :param config: Configuration object :type config: interfaces.IConfig :param unused_plugins: List of plugins (deprecated) - :type unused_plugins: `list` of `str` + :type unused_plugins: plugins_disco.PluginsRegistry :returns: `None` or string indicating error in case of error :rtype: None or str @@ -1035,23 +1088,32 @@ def revoke(config, unused_plugins): config.installer = config.authenticator = None if config.cert_path is None and config.certname: - config.cert_path = storage.cert_path_for_cert_name(config, config.certname) + # When revoking via --cert-name, take the cert path and server from renewalparams + lineage = storage.RenewableCert( + storage.renewal_file_for_certname(config, config.certname), config) + config.cert_path = lineage.cert_path + # --server takes priority over lineage.server + if lineage.server and not cli.set_by_cli("server"): + config.server = lineage.server elif not config.cert_path or (config.cert_path and config.certname): # intentionally not supporting --cert-path & --cert-name together, # to avoid dealing with mismatched values raise errors.Error("Error! Exactly one of --cert-path or --cert-name must be specified!") if config.key_path is not None: # revocation by cert key - logger.debug("Revoking %s using cert key %s", - config.cert_path[0], config.key_path[0]) - crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) - key = jose.JWK.load(config.key_path[1]) + logger.debug("Revoking %s using certificate key %s", + config.cert_path, config.key_path) + crypto_util.verify_cert_matches_priv_key(config.cert_path, config.key_path) + with open(config.key_path, 'rb') as f: + key = jose.JWK.load(f.read()) acme = client.acme_from_config_key(config, key) else: # revocation by account key - logger.debug("Revoking %s using Account Key", config.cert_path[0]) + logger.debug("Revoking %s using Account Key", config.cert_path) acc, _ = _determine_account(config) acme = client.acme_from_config_key(config, acc.key, acc.regr) - cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] + + with open(config.cert_path, 'rb') as f: + cert = crypto_util.pyopenssl_load_certificate(f.read())[0] logger.debug("Reason code for revocation: %s", config.reason) try: acme.revoke(jose.ComparableX509(cert), config.reason) @@ -1059,7 +1121,7 @@ def revoke(config, unused_plugins): except acme_errors.ClientError as e: return str(e) - display_ops.success_revocation(config.cert_path[0]) + display_ops.success_revocation(config.cert_path) return None @@ -1070,7 +1132,7 @@ def run(config, plugins): :type config: interfaces.IConfig :param plugins: List of plugins - :type plugins: `list` of `str` + :type plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -1149,6 +1211,7 @@ def _csr_get_and_save_cert(config, le_client): os.path.normpath(config.chain_path), os.path.normpath(config.fullchain_path)) return cert_path, fullchain_path + def renew_cert(config, plugins, lineage): """Renew & save an existing cert. Do not install it. @@ -1156,7 +1219,7 @@ def renew_cert(config, plugins, lineage): :type config: interfaces.IConfig :param plugins: List of plugins - :type plugins: `list` of `str` + :type plugins: plugins_disco.PluginsRegistry :param lineage: Certificate lineage object :type lineage: storage.RenewableCert @@ -1201,7 +1264,7 @@ def certonly(config, plugins): :type config: interfaces.IConfig :param plugins: List of plugins - :type plugins: `list` of `str` + :type plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -1251,7 +1314,7 @@ def renew(config, unused_plugins): :type config: interfaces.IConfig :param unused_plugins: List of plugins (deprecated) - :type unused_plugins: `list` of `str` + :type unused_plugins: plugins_disco.PluginsRegistry :returns: `None` :rtype: None @@ -1325,6 +1388,7 @@ def main(cli_args=None): plugins = plugins_disco.PluginsRegistry.find_all() logger.debug("certbot version: %s", certbot.__version__) + logger.debug("Location of certbot entry point: %s", sys.argv[0]) # do not log `config`, as it contains sensitive data (e.g. revoke --key)! logger.debug("Arguments: %r", cli_args) logger.debug("Discovered plugins: %r", plugins) diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index bed22b4ae..1e1e00982 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -1,11 +1,10 @@ """Utilities for plugins discovery and selection.""" -import collections import itertools import logging import sys +from collections.abc import Mapping import pkg_resources -import six import zope.interface import zope.interface.verify @@ -15,12 +14,6 @@ from certbot import interfaces from certbot._internal import constants from certbot.compat import os -try: - # Python 3.3+ - from collections.abc import Mapping -except ImportError: # pragma: no cover - from collections import Mapping - logger = logging.getLogger(__name__) PREFIX_FREE_DISTRIBUTIONS = [ @@ -45,7 +38,7 @@ PREFIX_FREE_DISTRIBUTIONS = [ """Distributions for which prefix will be omitted.""" -class PluginEntryPoint(object): +class PluginEntryPoint: """Plugin entry point.""" # this object is mutable, don't allow it to be hashed! @@ -215,7 +208,7 @@ class PluginsRegistry(Mapping): # This prevents deadlock caused by plugins acquiring a lock # and ensures at least one concurrent Certbot instance will run # successfully. - self._plugins = collections.OrderedDict(sorted(six.iteritems(plugins))) + self._plugins = dict(sorted(plugins.items())) @classmethod def find_all(cls): @@ -276,12 +269,12 @@ class PluginsRegistry(Mapping): def init(self, config): """Initialize all plugins in the registry.""" return [plugin_ep.init(config) for plugin_ep - in six.itervalues(self._plugins)] + in self._plugins.values()] def filter(self, pred): """Filter plugins based on predicate.""" return type(self)({name: plugin_ep for name, plugin_ep - in six.iteritems(self._plugins) if pred(plugin_ep)}) + in self._plugins.items() if pred(plugin_ep)}) def visible(self): """Filter plugins based on visibility.""" @@ -297,7 +290,7 @@ class PluginsRegistry(Mapping): def prepare(self): """Prepare all plugins in the registry.""" - return [plugin_ep.prepare() for plugin_ep in six.itervalues(self._plugins)] + return [plugin_ep.prepare() for plugin_ep in self._plugins.values()] def available(self): """Filter plugins based on availability.""" @@ -319,7 +312,7 @@ class PluginsRegistry(Mapping): """ # use list instead of set because PluginEntryPoint is not hashable - candidates = [plugin_ep for plugin_ep in six.itervalues(self._plugins) + candidates = [plugin_ep for plugin_ep in self._plugins.values() if plugin_ep.initialized and plugin_ep.init() is plugin] assert len(candidates) <= 1 if candidates: @@ -329,9 +322,9 @@ class PluginsRegistry(Mapping): def __repr__(self): return "{0}({1})".format( self.__class__.__name__, ','.join( - repr(p_ep) for p_ep in six.itervalues(self._plugins))) + repr(p_ep) for p_ep in self._plugins.values())) def __str__(self): if not self._plugins: return "No plugins" - return "\n\n".join(str(p_ep) for p_ep in six.itervalues(self._plugins)) + return "\n\n".join(str(p_ep) for p_ep in self._plugins.values()) diff --git a/certbot/certbot/_internal/plugins/null.py b/certbot/certbot/_internal/plugins/null.py index cf7c05a2b..5ab2f4a04 100644 --- a/certbot/certbot/_internal/plugins/null.py +++ b/certbot/certbot/_internal/plugins/null.py @@ -1,7 +1,6 @@ """Null plugin.""" import logging -import zope.component import zope.interface from certbot import interfaces diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index 0b04791c6..bf7e505b9 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -1,9 +1,7 @@ """Decide which plugins to use for authentication & installation""" -from __future__ import print_function import logging -import six import zope.component from certbot import errors @@ -108,7 +106,7 @@ def pick_plugin(config, default, plugins, question, ifaces): if len(prepared) > 1: logger.debug("Multiple candidate plugins: %s", prepared) - plugin_ep = choose_plugin(list(six.itervalues(prepared)), question) + plugin_ep = choose_plugin(list(prepared.values()), question) if plugin_ep is None: return None return plugin_ep.init() diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index bbb56178c..d5d9fd2ec 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -6,7 +6,6 @@ import socket from socket import errno as socket_errors # type: ignore import OpenSSL # pylint: disable=unused-import -import six import zope.interface from acme import challenges @@ -29,7 +28,7 @@ if TYPE_CHECKING: Set[achallenges.KeyAuthorizationAnnotatedChallenge] ] -class ServerManager(object): +class ServerManager: """Standalone servers manager. Manager for `ACMEServer` and `ACMETLSServer` instances. @@ -183,7 +182,7 @@ class Authenticator(common.Plugin): for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, servers in six.iteritems(self.servers.running()): + for port, servers in self.servers.running().items(): if not self.served[servers]: self.servers.stop(port) diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index 484d209d6..8789db604 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -4,7 +4,6 @@ import collections import json import logging -import six import zope.component import zope.interface @@ -92,7 +91,7 @@ to serve all files under specified web root ({0}).""" for achall in achalls: self.conf("map").setdefault(achall.domain, webroot_path) else: - known_webroots = list(set(six.itervalues(self.conf("map")))) + known_webroots = list(set(self.conf("map").values())) for achall in achalls: if achall.domain not in self.conf("map"): new_webroot = self._prompt_for_webroot(achall.domain, @@ -157,7 +156,8 @@ to serve all files under specified web root ({0}).""" "--webroot-path and --domains, or --webroot-map. Run with " " --help webroot for examples.") for name, path in path_map.items(): - self.full_roots[name] = os.path.join(path, challenges.HTTP01.URI_ROOT_PATH) + self.full_roots[name] = os.path.join(path, os.path.normcase( + challenges.HTTP01.URI_ROOT_PATH)) logger.debug("Creating root challenges validation dir at %s", self.full_roots[name]) @@ -241,7 +241,7 @@ class _WebrootMapAction(argparse.Action): """Action class for parsing webroot_map.""" def __call__(self, parser, namespace, webroot_map, option_string=None): - for domains, webroot_path in six.iteritems(json.loads(webroot_map)): + for domains, webroot_path in json.loads(webroot_map).items(): webroot_path = _validate_webroot(webroot_path) namespace.webroot_map.update( (d, webroot_path) for d in cli.add_domains(namespace, domains)) diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index 50e4d3a4c..0b99ae3d1 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -1,5 +1,4 @@ """Functionality for autorenewal and associated juggling of configurations""" -from __future__ import print_function import copy import itertools @@ -10,15 +9,15 @@ import time import traceback from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.asymmetric import ec, rsa from cryptography.hazmat.primitives.serialization import load_pem_private_key import OpenSSL -import six import zope.component from acme.magic_typing import List from acme.magic_typing import Optional # pylint: disable=unused-import from certbot import crypto_util +from certbot.display import util as display_util from certbot import errors from certbot import interfaces from certbot import util @@ -40,7 +39,7 @@ logger = logging.getLogger(__name__) STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", "server", "account", "authenticator", "installer", "renew_hook", "pre_hook", "post_hook", "http01_address", - "preferred_chain"] + "preferred_chain", "key_type", "elliptic_curve"] INT_CONFIG_ITEMS = ["rsa_key_size", "http01_port"] BOOL_CONFIG_ITEMS = ["must_staple", "allow_subset_of_names", "reuse_key", "autorenew"] @@ -84,6 +83,7 @@ def _reconstitute(config, full_path): return None # Now restore specific values along with their data types, if # those elements are present. + renewalparams = _remove_deprecated_config_elements(renewalparams) try: restore_required_config_elements(config, renewalparams) _restore_plugin_configs(config, renewalparams) @@ -98,7 +98,7 @@ def _reconstitute(config, full_path): config.domains = [util.enforce_domain_sanity(d) for d in renewal_candidate.names()] except errors.ConfigurationError as error: - logger.warning("Renewal configuration file %s references a cert " + logger.warning("Renewal configuration file %s references a certificate " "that contains an invalid domain name. The problem " "was: %s. Skipping.", full_path, error) return None @@ -118,7 +118,7 @@ def _restore_webroot_config(config, renewalparams): # see https://github.com/certbot/certbot/pull/7095 if "webroot_path" in renewalparams and not cli.set_by_cli("webroot_path"): wp = renewalparams["webroot_path"] - if isinstance(wp, six.string_types): # prior to 0.1.0, webroot_path was a string + if isinstance(wp, str): # prior to 0.1.0, webroot_path was a string wp = [wp] config.webroot_path = wp @@ -153,7 +153,7 @@ def _restore_plugin_configs(config, renewalparams): for plugin_prefix in set(plugin_prefixes): plugin_prefix = plugin_prefix.replace('-', '_') - for config_item, config_value in six.iteritems(renewalparams): + for config_item, config_value in renewalparams.items(): if config_item.startswith(plugin_prefix + "_") and not cli.set_by_cli(config_item): # Values None, True, and False need to be treated specially, # As their types aren't handled correctly by configobj @@ -178,15 +178,28 @@ def restore_required_config_elements(config, renewalparams): required_items = itertools.chain( (("pref_challs", _restore_pref_challs),), - six.moves.zip(BOOL_CONFIG_ITEMS, itertools.repeat(_restore_bool)), - six.moves.zip(INT_CONFIG_ITEMS, itertools.repeat(_restore_int)), - six.moves.zip(STR_CONFIG_ITEMS, itertools.repeat(_restore_str))) + zip(BOOL_CONFIG_ITEMS, itertools.repeat(_restore_bool)), + zip(INT_CONFIG_ITEMS, itertools.repeat(_restore_int)), + zip(STR_CONFIG_ITEMS, itertools.repeat(_restore_str))) for item_name, restore_func in required_items: if item_name in renewalparams and not cli.set_by_cli(item_name): value = restore_func(item_name, renewalparams[item_name]) setattr(config, item_name, value) +def _remove_deprecated_config_elements(renewalparams): + """Removes deprecated config options from the parsed renewalparams. + + :param dict renewalparams: list of parsed renewalparams + + :returns: list of renewalparams with deprecated config options removed + :rtype: dict + + """ + return {option_name: v for (option_name, v) in renewalparams.items() + if option_name not in cli.DEPRECATED_OPTIONS} + + def _restore_pref_challs(unused_name, value): """Restores preferred challenges from a renewal config file. @@ -205,7 +218,7 @@ def _restore_pref_challs(unused_name, value): # If pref_challs has only one element, configobj saves the value # with a trailing comma so it's parsed as a list. If this comma is # removed by the user, the value is parsed as a str. - value = [value] if isinstance(value, six.string_types) else value + value = [value] if isinstance(value, str) else value return cli.parse_preferred_challenges(value) @@ -292,18 +305,15 @@ def should_renew(config, lineage): def _avoid_invalidating_lineage(config, lineage, original_server): "Do not renew a valid cert with one from a staging server!" - # Some lineages may have begun with --staging, but then had production certs - # added to them + # Some lineages may have begun with --staging, but then had production + # certificates added to them with open(lineage.cert) as the_file: contents = the_file.read() latest_cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, contents) - # all our test certs are from happy hacker fake CA, though maybe one day - # we should test more methodically - now_valid = "fake" not in repr(latest_cert.get_issuer()).lower() if util.is_staging(config.server): - if not util.is_staging(original_server) or now_valid: + if not util.is_staging(original_server): if not config.break_my_certs: names = ", ".join(lineage.names()) raise errors.Error( @@ -347,40 +357,42 @@ def report(msgs, category): def _renew_describe_results(config, renew_successes, renew_failures, renew_skipped, parse_failures): + # type: (interfaces.IConfig, List[str], List[str], List[str], List[str]) -> None + """ + Print a report to the terminal about the results of the renewal process. - out = [] # type: List[str] - notify = out.append - disp = zope.component.getUtility(interfaces.IDisplay) + :param interfaces.IConfig config: Configuration + :param list renew_successes: list of fullchain paths which were renewed + :param list renew_failures: list of fullchain paths which failed to be renewed + :param list renew_skipped: list of messages to print about skipped certificates + :param list parse_failures: list of renewal parameter paths which had erorrs + """ + notify = display_util.notify + notify_error = logger.error - def notify_error(err): - """Notify and log errors.""" - notify(str(err)) - logger.error(err) + notify('\n{}'.format(display_util.SIDE_FRAME)) + + renewal_noun = "simulated renewal" if config.dry_run else "renewal" - if config.dry_run: - notify("** DRY RUN: simulating 'certbot renew' close to cert expiry") - notify("** (The test certificates below have not been saved.)") - notify("") if renew_skipped: - notify("The following certs are not due for renewal yet:") + notify("The following certificates are not due for renewal yet:") notify(report(renew_skipped, "skipped")) if not renew_successes and not renew_failures: - notify("No renewals were attempted.") + notify("No {renewal}s were attempted.".format(renewal=renewal_noun)) if (config.pre_hook is not None or config.renew_hook is not None or config.post_hook is not None): notify("No hooks were run.") elif renew_successes and not renew_failures: - notify("Congratulations, all renewals succeeded. The following certs " - "have been renewed:") + notify("Congratulations, all {renewal}s succeeded: ".format(renewal=renewal_noun)) notify(report(renew_successes, "success")) elif renew_failures and not renew_successes: - notify_error("All renewal attempts failed. The following certs could " - "not be renewed:") + notify_error("All %ss failed. The following certificates could " + "not be renewed:", renewal_noun) notify_error(report(renew_failures, "failure")) elif renew_failures and renew_successes: - notify("The following certs were successfully renewed:") + notify("The following {renewal}s succeeded:".format(renewal=renewal_noun)) notify(report(renew_successes, "success") + "\n") - notify_error("The following certs could not be renewed:") + notify_error("The following %ss failed:", renewal_noun) notify_error(report(renew_failures, "failure")) if parse_failures: @@ -388,11 +400,7 @@ def _renew_describe_results(config, renew_successes, renew_failures, "were invalid: ") notify(report(parse_failures, "parsefail")) - if config.dry_run: - notify("** DRY RUN: simulating 'certbot renew' close to cert expiry") - notify("** (The test certificates above have not been saved.)") - - disp.notification("\n".join(out), wrap=False) + notify(display_util.SIDE_FRAME) def handle_renewal_request(config): @@ -482,9 +490,10 @@ def handle_renewal_request(config): except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. - logger.warning("Attempting to renew cert (%s) from %s produced an " - "unexpected error: %s. Skipping.", lineagename, - renewal_file, e) + logger.error( + "Failed to renew certificate %s with error: %s", + lineagename, e + ) logger.debug("Traceback was:\n%s", traceback.format_exc()) renew_failures.append(renewal_candidate.fullchain) @@ -506,6 +515,10 @@ def _update_renewal_params_from_key(key_path, config): with open(key_path, 'rb') as file_h: key = load_pem_private_key(file_h.read(), password=None, backend=default_backend()) if isinstance(key, rsa.RSAPrivateKey): + config.key_type = 'rsa' config.rsa_key_size = key.key_size + elif isinstance(key, ec.EllipticCurvePrivateKey): + config.key_type = 'ecdsa' + config.elliptic_curve = key.curve.name else: raise errors.Error('Key at {0} is of an unsupported type: {1}.'.format(key_path, type(key))) diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index d781f3d9d..3093ef8ca 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -1,12 +1,10 @@ """Collects and displays information to the user.""" -from __future__ import print_function - import collections import logging +import queue import sys import textwrap -from six.moves import queue # type: ignore import zope.interface from certbot import interfaces @@ -16,7 +14,7 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IReporter) -class Reporter(object): +class Reporter: """Collects and displays information to the user. :ivar `queue.PriorityQueue` messages: Messages to be displayed to diff --git a/certbot/certbot/_internal/snap_config.py b/certbot/certbot/_internal/snap_config.py index e92d9431c..a90740787 100644 --- a/certbot/certbot/_internal/snap_config.py +++ b/certbot/certbot/_internal/snap_config.py @@ -49,22 +49,22 @@ def prepare_env(cli_args): os.environ['CERTBOT_AUGEAS_PATH'] = '{0}/usr/lib/{1}/libaugeas.so.0'.format( os.environ.get('SNAP'), _ARCH_TRIPLET_MAP[snap_arch]) - session = Session() - session.mount('http://snapd/', _SnapdAdapter()) + with Session() as session: + session.mount('http://snapd/', _SnapdAdapter()) - try: - response = session.get('http://snapd/v2/connections?snap=certbot&interface=content') - response.raise_for_status() - except RequestException as e: - if isinstance(e, HTTPError) and e.response.status_code == 404: - LOGGER.error('An error occurred while fetching Certbot snap plugins: ' - 'your version of snapd is outdated.') - LOGGER.error('Please run "sudo snap install core; sudo snap refresh core" ' - 'in your terminal and try again.') - else: - LOGGER.error('An error occurred while fetching Certbot snap plugins: ' - 'make sure the snapd service is running.') - raise e + try: + response = session.get('http://snapd/v2/connections?snap=certbot&interface=content') + response.raise_for_status() + except RequestException as e: + if isinstance(e, HTTPError) and e.response.status_code == 404: + LOGGER.error('An error occurred while fetching Certbot snap plugins: ' + 'your version of snapd is outdated.') + LOGGER.error('Please run "sudo snap install core; sudo snap refresh core" ' + 'in your terminal and try again.') + else: + LOGGER.error('An error occurred while fetching Certbot snap plugins: ' + 'make sure the snapd service is running.') + raise e data = response.json() connections = ['/snap/{0}/current/lib/python3.8/site-packages/'.format(item['slot']['snap']) diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 84bd33143..218a5dd92 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -6,10 +6,13 @@ import re import shutil import stat +from typing import Optional import configobj import parsedatetime import pytz -import six +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey +from cryptography.hazmat.primitives.serialization import load_pem_private_key import certbot from certbot import crypto_util @@ -46,6 +49,7 @@ def renewal_conf_files(config): result.sort() return result + def renewal_file_for_certname(config, certname): """Return /path/to/certname.conf in the renewal conf directory""" path = os.path.join(config.renewal_configs_dir, "{0}.conf".format(certname)) @@ -55,7 +59,7 @@ def renewal_file_for_certname(config, certname): return path -def cert_path_for_cert_name(config, cert_name): +def cert_path_for_cert_name(config: interfaces.IConfig, cert_name: str) -> str: """ If `--cert-name` was specified, but you need a value for `--cert-path`. :param `configuration.NamespaceConfig` config: parsed command line arguments @@ -63,10 +67,7 @@ def cert_path_for_cert_name(config, cert_name): """ cert_name_implied_conf = renewal_file_for_certname(config, cert_name) - fullchain_path = configobj.ConfigObj(cert_name_implied_conf)["fullchain"] - with open(fullchain_path) as f: - cert_path = (fullchain_path, f.read()) - return cert_path + return configobj.ConfigObj(cert_name_implied_conf)["fullchain"] def config_with_defaults(config=None): @@ -210,7 +211,7 @@ def get_link_target(link): """ try: - target = os.readlink(link) + target = filesystem.readlink(link) except OSError: raise errors.CertStorageError( "Expected {0} to be a symlink".format(link)) @@ -219,6 +220,7 @@ def get_link_target(link): target = os.path.join(os.path.dirname(link), target) return os.path.abspath(target) + def _write_live_readme_to(readme_path, is_base_dir=False): prefix = "" if is_base_dir: @@ -270,7 +272,7 @@ def relevant_values(all_values): rv = dict( (option, value) - for option, value in six.iteritems(all_values) + for option, value in all_values.items() if _relevant(namespaces, option) and cli.option_was_set(option, value)) # We always save the server value to help with forward compatibility # and behavioral consistency when versions of Certbot with different @@ -517,11 +519,15 @@ class RenewableCert(interfaces.RenewableCert): return _relpath_from_file(self.archive_dir, from_file) @property - def is_test_cert(self): + def server(self) -> Optional[str]: + """Returns the ACME server associated with this certificate""" + return self.configuration["renewalparams"].get("server", None) + + @property + def is_test_cert(self) -> bool: """Returns true if this is a test cert from a staging server.""" - server = self.configuration["renewalparams"].get("server", None) - if server: - return util.is_staging(server) + if self.server: + return util.is_staging(self.server) return False def _check_symlinks(self): @@ -661,7 +667,7 @@ class RenewableCert(interfaces.RenewableCert): current_link = getattr(self, kind) if os.path.lexists(current_link): os.unlink(current_link) - os.symlink(os.readlink(previous_link), current_link) + os.symlink(filesystem.readlink(previous_link), current_link) for _, link in previous_symlinks: if os.path.exists(link): @@ -805,8 +811,8 @@ class RenewableCert(interfaces.RenewableCert): May need to recover from rare interrupted / crashed states.""" if self.has_pending_deployment(): - logger.warning("Found a new cert /archive/ that was not linked to in /live/; " - "fixing...") + logger.warning("Found a new certificate /archive/ that was not " + "linked to in /live/; fixing...") self.update_all_links_to(self.latest_common_version()) return False return True @@ -842,7 +848,7 @@ class RenewableCert(interfaces.RenewableCert): link = getattr(self, kind) filename = "{0}{1}.pem".format(kind, version) # Relative rather than absolute target directory - target_directory = os.path.dirname(os.readlink(link)) + target_directory = os.path.dirname(filesystem.readlink(link)) # TODO: it could be safer to make the link first under a temporary # filename, then unlink the old link, then rename the new link # to the old link; this ensures that this process is able to @@ -879,7 +885,7 @@ class RenewableCert(interfaces.RenewableCert): """ target = self.current_target("cert") if target is None: - raise errors.CertStorageError("could not find cert file") + raise errors.CertStorageError("could not find the certificate file") with open(target) as f: return crypto_util.get_names_from_cert(f.read()) @@ -1055,6 +1061,23 @@ class RenewableCert(interfaces.RenewableCert): target, values) return cls(new_config.filename, cli_config) + @property + def private_key_type(self): + """ + :returns: The type of algorithm for the private, RSA or ECDSA + :rtype: str + """ + with open(self.configuration["privkey"], "rb") as priv_key_file: + key = load_pem_private_key( + data=priv_key_file.read(), + password=None, + backend=default_backend() + ) + if isinstance(key, RSAPrivateKey): + return "RSA" + else: + return "ECDSA" + def save_successor(self, prior_version, new_cert, new_privkey, new_chain, cli_config): """Save new cert and chain as a successor of a prior version. @@ -1100,7 +1123,7 @@ class RenewableCert(interfaces.RenewableCert): # The behavior below keeps the prior key by creating a new # symlink to the old key or the target of the old key symlink. if os.path.islink(old_privkey): - old_privkey = os.readlink(old_privkey) + old_privkey = filesystem.readlink(old_privkey) else: old_privkey = "privkey{0}.pem".format(prior_version) logger.debug("Writing symlink to old private key, %s.", old_privkey) diff --git a/certbot/certbot/_internal/updater.py b/certbot/certbot/_internal/updater.py index 961436ca5..23ba06da3 100644 --- a/certbot/certbot/_internal/updater.py +++ b/certbot/certbot/_internal/updater.py @@ -18,7 +18,7 @@ def run_generic_updaters(config, lineage, plugins): :type lineage: storage.RenewableCert :param plugins: List of plugins - :type plugins: `list` of `str` + :type plugins: certbot._internal.plugins.disco.PluginsRegistry :returns: `None` :rtype: None diff --git a/certbot/certbot/achallenges.py b/certbot/certbot/achallenges.py index 70588683d..7171c271c 100644 --- a/certbot/certbot/achallenges.py +++ b/certbot/certbot/achallenges.py @@ -33,7 +33,7 @@ class AnnotatedChallenge(jose.ImmutableMap): Wraps around server provided challenge and annotates with data useful for the client. - :ivar challb: Wrapped `~.ChallengeBody`. + :ivar ~.challb: Wrapped `~.ChallengeBody`. """ __slots__ = ('challb',) diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index 0c8a7514b..0152685e9 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import errno import os # pylint: disable=os-module-forbidden import stat +import sys from acme.magic_typing import List @@ -361,7 +362,8 @@ def realpath(file_path): """ original_path = file_path - if POSIX_MODE: + # Since Python 3.8, os.path.realpath also resolves symlinks on Windows. + if POSIX_MODE or sys.version_info >= (3, 8): path = os.path.realpath(file_path) if os.path.islink(path): # If path returned by realpath is still a link, it means that it failed to @@ -383,8 +385,36 @@ def realpath(file_path): return os.path.abspath(file_path) +def readlink(link_path): + # type: (str) -> str + """ + Return a string representing the path to which the symbolic link points. + + :param str link_path: The symlink path to resolve + :return: The path the symlink points to + :returns: str + :raise: ValueError if a long path (260> characters) is encountered on Windows + """ + path = os.readlink(link_path) + + if POSIX_MODE or not path.startswith('\\\\?\\'): + return path + + # At this point, we know we are on Windows and that the path returned uses + # the extended form which is done for all paths in Python 3.8+ + + # Max length of a normal path is 260 characters on Windows, including the non printable + # termination character "". The termination character is not included in Python + # strings, giving a max length of 259 characters, + 4 characters for the extended form + # prefix, to an effective max length 263 characters on a string representing a normal path. + if len(path) < 264: + return path[4:] + + raise ValueError("Long paths are not supported by Certbot on Windows.") + + # On Windows is_executable run from an unprivileged shell may claim that a path is -# executable when it is excutable only if run from a privileged shell. This result +# executable when it is executable only if run from a privileged shell. This result # is due to the fact that GetEffectiveRightsFromAcl calculate effective rights # without taking into consideration if the target user has currently required the # elevated privileges or not. However this is not a problem since certbot always diff --git a/certbot/certbot/compat/os.py b/certbot/certbot/compat/os.py index b4aea054f..2f0899bd7 100644 --- a/certbot/certbot/compat/os.py +++ b/certbot/certbot/compat/os.py @@ -7,6 +7,10 @@ This module has the same API as the os module in the Python standard library except for the functions defined below. """ + +# NOTE: If adding a new documented function to compat.os, ensure that it is added to the +# ':members:' list in certbot/docs/api/certbot.compat.os.rst. + # isort:skip_file # pylint: disable=function-redefined from __future__ import absolute_import @@ -152,3 +156,14 @@ def fstat(*unused_args, **unused_kwargs): raise RuntimeError('Usage of os.fstat() is forbidden. ' 'Use certbot.compat.filesystem functions instead ' '(eg. has_min_permissions, has_same_ownership).') + + +# Method os.readlink has a significant behavior change with Python 3.8+. Starting +# with this version, it will return the resolved path in its "extended-style" form +# unconditionally, which allows to use more than 259 characters, and its string +# representation is prepended with "\\?\". Problem is that it does it for any path, +# and will make equality comparison fail with paths that will use the simple form. +def readlink(*unused_args, **unused_kwargs): + """Method os.readlink() is forbidden""" + raise RuntimeError('Usage of os.readlink() is forbidden. ' + 'Use certbot.compat.filesystem.realpath() instead.') diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 5de412b5e..f67d95d97 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -11,16 +11,17 @@ import warnings import re # See https://github.com/pyca/cryptography/issues/4275 from cryptography import x509 # type: ignore -from cryptography.exceptions import InvalidSignature +from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric.ec import ECDSA -from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA, EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat from OpenSSL import crypto from OpenSSL import SSL # type: ignore + import pyrfc3339 -import six import zope.component from acme import crypto_util as acme_crypto_util @@ -34,7 +35,9 @@ logger = logging.getLogger(__name__) # High level functions -def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): +def init_save_key( + key_size, key_dir, key_type="rsa", elliptic_curve="secp256r1", keyname="key-certbot.pem" +): """Initializes and saves a privkey. Inits key and saves it in PEM format on the filesystem. @@ -42,8 +45,10 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): .. note:: keyname is the attempted filename, it may be different if a file already exists at the path. - :param int key_size: RSA key size in bits + :param int key_size: key size in bits if key size is rsa. :param str key_dir: Key save directory. + :param str key_type: Key Type [rsa, ecdsa] + :param str elliptic_curve: Name of the elliptic curve if key type is ecdsa. :param str keyname: Filename of key :returns: Key @@ -53,7 +58,9 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): """ try: - key_pem = make_key(key_size) + key_pem = make_key( + bits=key_size, elliptic_curve=elliptic_curve or "secp256r1", key_type=key_type, + ) except ValueError as err: logger.error("", exc_info=True) raise err @@ -65,7 +72,10 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): os.path.join(key_dir, keyname), 0o600, "wb") with key_f: key_f.write(key_pem) - logger.debug("Generating key (%d bits): %s", key_size, key_path) + if key_type == 'rsa': + logger.debug("Generating RSA key (%d bits): %s", key_size, key_path) + else: + logger.debug("Generating ECDSA key (%d bits): %s", key_size, key_path) return util.Key(key_path, key_pem) @@ -174,18 +184,45 @@ def import_csr_file(csrfile, data): return PEM, util.CSR(file=csrfile, data=data_pem, form="pem"), domains -def make_key(bits): - """Generate PEM encoded RSA key. +def make_key(bits=1024, key_type="rsa", elliptic_curve=None): + """Generate PEM encoded RSA|EC key. - :param int bits: Number of bits, at least 1024. + :param int bits: Number of bits if key_type=rsa. At least 1024 for RSA. - :returns: new RSA key in PEM form with specified number of bits + :param str ec_curve: The elliptic curve to use. + + :returns: new RSA or ECDSA key in PEM form with specified number of bits + or of type ec_curve when key_type ecdsa is used. :rtype: str - """ - assert bits >= 1024 # XXX - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, bits) + if key_type == 'rsa': + if bits < 1024: + raise errors.Error("Unsupported RSA key length: {}".format(bits)) + + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) + elif key_type == 'ecdsa': + try: + name = elliptic_curve.upper() + if name in ('SECP256R1', 'SECP384R1', 'SECP521R1'): + _key = ec.generate_private_key( + curve=getattr(ec, elliptic_curve.upper(), None)(), + backend=default_backend() + ) + else: + raise errors.Error("Unsupported elliptic curve: {}".format(elliptic_curve)) + except TypeError: + raise errors.Error("Unsupported elliptic curve: {}".format(elliptic_curve)) + except UnsupportedAlgorithm as e: + raise e from errors.Error(str(e)) + _key_pem = _key.private_bytes( + encoding=Encoding.PEM, + format=PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=NoEncryption() + ) + key = crypto.load_privatekey(crypto.FILETYPE_PEM, _key_pem) + else: + raise errors.Error("Invalid key_type specified: {}. Use [rsa|ecdsa]".format(key_type)) return crypto.dump_privatekey(crypto.FILETYPE_PEM, key) @@ -241,7 +278,7 @@ def verify_renewable_cert_sig(renewable_cert): verify_signed_payload(pk, cert.signature, cert.tbs_certificate_bytes, cert.signature_hash_algorithm) except (IOError, ValueError, InvalidSignature) as e: - error_str = "verifying the signature of the cert located at {0} has failed. \ + error_str = "verifying the signature of the certificate located at {0} has failed. \ Details: {1}".format(renewable_cert.cert_path, e) logger.exception(error_str) raise errors.Error(error_str) @@ -253,7 +290,7 @@ def verify_signed_payload(public_key, signature, payload, signature_hash_algorit :param RSAPublicKey/EllipticCurvePublicKey public_key: the public_key to check signature :param bytes signature: the signature bytes :param bytes payload: the payload bytes - :param cryptography.hazmat.primitives.hashes.HashAlgorithm + :param cryptography.hazmat.primitives.hashes.HashAlgorithm \ signature_hash_algorithm: algorithm used to hash the payload :raises InvalidSignature: If signature verification fails. @@ -292,7 +329,7 @@ def verify_cert_matches_priv_key(cert_path, key_path): context.use_privatekey_file(key_path) context.check_privatekey() except (IOError, SSL.Error) as e: - error_str = "verifying the cert located at {0} matches the \ + error_str = "verifying the certificate located at {0} matches the \ private key located at {1} has failed. \ Details: {2}".format(cert_path, key_path, e) @@ -447,22 +484,16 @@ def _notAfterBefore(cert_path, method): """ # pylint: disable=redefined-outer-name - with open(cert_path) as f: - x509 = crypto.load_certificate(crypto.FILETYPE_PEM, - f.read()) + with open(cert_path, "rb") as f: # type: IO[bytes] + x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) # pyopenssl always returns bytes timestamp = method(x509) reformatted_timestamp = [timestamp[0:4], b"-", timestamp[4:6], b"-", timestamp[6:8], b"T", timestamp[8:10], b":", timestamp[10:12], b":", timestamp[12:]] - # pyrfc3339 always uses the type `str`. This means that in Python 2, it - # expects str/bytes and in Python 3 it expects its str type or the Python 2 - # equivalent of the type unicode. + # pyrfc3339 always uses the type `str` timestamp_bytes = b"".join(reformatted_timestamp) - if six.PY3: - timestamp_str = timestamp_bytes.decode('ascii') - else: - timestamp_str = timestamp_bytes + timestamp_str = timestamp_bytes.decode('ascii') return pyrfc3339.parse(timestamp_str) @@ -520,6 +551,7 @@ def cert_and_chain_from_fullchain(fullchain_pem): # Since each normalized cert has a newline suffix, no extra newlines are required. return (certs_normalized[0], "".join(certs_normalized[1:])) + def get_serial_from_cert(cert_path): """Retrieve the serial number of a certificate from certificate path @@ -529,15 +561,15 @@ def get_serial_from_cert(cert_path): :rtype: int """ # pylint: disable=redefined-outer-name - with open(cert_path) as f: - x509 = crypto.load_certificate(crypto.FILETYPE_PEM, - f.read()) + with open(cert_path, "rb") as f: # type: IO[bytes] + x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) return x509.get_serial_number() def find_chain_with_issuer(fullchains, issuer_cn, warn_on_no_match=False): - """Chooses the first certificate chain from fullchains which contains an - Issuer Subject Common Name matching issuer_cn. + """Chooses the first certificate chain from fullchains whose topmost + intermediate has an Issuer Common Name matching issuer_cn (in other words + the first chain which chains to a root whose name matches issuer_cn). :param fullchains: The list of fullchains in PEM chain format. :type fullchains: `list` of `str` @@ -548,14 +580,11 @@ def find_chain_with_issuer(fullchains, issuer_cn, warn_on_no_match=False): :rtype: `str` """ for chain in fullchains: - certs = [x509.load_pem_x509_certificate(cert, default_backend()) \ - for cert in CERT_PEM_REGEX.findall(chain.encode())] - # Iterate the fullchain beginning from the leaf. For each certificate encountered, - # match against Issuer Subject CN. - for cert in certs: - cert_issuer_cn = cert.issuer.get_attributes_for_oid(x509.NameOID.COMMON_NAME) - if cert_issuer_cn and cert_issuer_cn[0].value == issuer_cn: - return chain + certs = CERT_PEM_REGEX.findall(chain.encode()) + top_cert = x509.load_pem_x509_certificate(certs[-1], default_backend()) + top_issuer_cn = top_cert.issuer.get_attributes_for_oid(x509.NameOID.COMMON_NAME) + if top_issuer_cn and top_issuer_cn[0].value == issuer_cn: + return chain # Nothing matched, return whatever was first in the list. if warn_on_no_match: diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index bc56f8778..f26ec468f 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -16,6 +16,7 @@ import textwrap import zope.interface import zope.component +from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot._internal import constants @@ -70,7 +71,7 @@ def _wrap_lines(msg): def input_with_timeout(prompt=None, timeout=36000.0): """Get user input with a timeout. - Behaves the same as six.moves.input, however, an error is raised if + Behaves the same as the builtin input, however, an error is raised if a user doesn't answer after timeout seconds. The default timeout value was chosen to place it just under 12 hours for users following our advice and running Certbot twice a day. @@ -84,7 +85,7 @@ def input_with_timeout(prompt=None, timeout=36000.0): :raises errors.Error if no answer is given before the timeout """ - # use of sys.stdin and sys.stdout to mimic six.moves.input based on + # use of sys.stdin and sys.stdout to mimic the builtin input based on # https://github.com/python/cpython/blob/baf7bb30a02aabde260143136bdf5b3738a1d409/Lib/getpass.py#L129 if prompt: sys.stdout.write(prompt) @@ -105,12 +106,12 @@ def notify(msg): """ zope.component.getUtility(interfaces.IDisplay).notification( - msg, pause=False, decorate=False + msg, pause=False, decorate=False, wrap=False ) @zope.interface.implementer(interfaces.IDisplay) -class FileDisplay(object): +class FileDisplay: """File-based display.""" # see https://github.com/certbot/certbot/issues/3915 @@ -477,7 +478,7 @@ def assert_valid_call(prompt, default, cli_flag, force_interactive): @zope.interface.implementer(interfaces.IDisplay) -class NoninteractiveDisplay(object): +class NoninteractiveDisplay: """An iDisplay implementation that never asks for interactive user input""" def __init__(self, outfile, *unused_args, **unused_kwargs): @@ -633,3 +634,28 @@ def _parens_around_char(label): """ return "({first}){rest}".format(first=label[0], rest=label[1:]) + + +def summarize_domain_list(domains): + # type: (List[str]) -> str + """Summarizes a list of domains in the format of: + example.com.com and N more domains + or if there is are only two domains: + example.com and www.example.com + or if there is only one domain: + example.com + + :param list domains: `str` list of domains + :returns: the domain list summary + :rtype: str + """ + if not domains: + return "" + + l = len(domains) + if l == 1: + return domains[0] + elif l == 2: + return " and ".join(domains) + else: + return "{0} and {1} more domains".format(domains[0], l-1) diff --git a/certbot/certbot/interfaces.py b/certbot/certbot/interfaces.py index 43f081b95..ddbee8ddc 100644 --- a/certbot/certbot/interfaces.py +++ b/certbot/certbot/interfaces.py @@ -1,14 +1,12 @@ """Certbot client interfaces.""" import abc -import six import zope.interface # pylint: disable=no-self-argument,no-method-argument,inherit-non-class -@six.add_metaclass(abc.ABCMeta) -class AccountStorage(object): +class AccountStorage(object, metaclass=abc.ABCMeta): """Accounts storage interface.""" @abc.abstractmethod @@ -198,6 +196,13 @@ class IConfig(zope.interface.Interface): "register multiple emails, ex: u1@example.com,u2@example.com. " "(default: Ask).") rsa_key_size = zope.interface.Attribute("Size of the RSA key.") + elliptic_curve = zope.interface.Attribute( + "The SECG elliptic curve name to use. Please see RFC 8446 " + "for supported values." + ) + key_type = zope.interface.Attribute( + "Type of generated private key" + "(Only *ONE* per invocation can be provided at this time)") must_staple = zope.interface.Attribute( "Adds the OCSP Must Staple extension to the certificate. " "Autoconfigures OCSP Stapling for supported setups " @@ -255,11 +260,12 @@ class IConfig(zope.interface.Interface): " with \"renew\" verb should be disabled.") preferred_chain = zope.interface.Attribute( - "If the CA offers multiple certificate chains, prefer the chain with " - "an issuer matching this Subject Common Name. If no match, the default " - "offered chain will be used." + "If the CA offers multiple certificate chains, prefer the chain whose " + "topmost certificate was issued from this Subject Common Name. " + "If no match, the default offered chain will be used." ) + class IInstaller(IPlugin): """Generic Certbot Installer Interface. @@ -539,8 +545,7 @@ class IReporter(zope.interface.Interface): """Prints messages to the user and clears the message queue.""" -@six.add_metaclass(abc.ABCMeta) -class RenewableCert(object): +class RenewableCert(object, metaclass=abc.ABCMeta): """Interface to a certificate lineage.""" @abc.abstractproperty @@ -605,8 +610,7 @@ class RenewableCert(object): # an update during the run or install subcommand, it should do so when # :func:`IInstaller.deploy_cert` is called. -@six.add_metaclass(abc.ABCMeta) -class GenericUpdater(object): +class GenericUpdater(object, metaclass=abc.ABCMeta): """Interface for update types not currently specified by Certbot. This class allows plugins to perform types of updates that Certbot hasn't @@ -638,8 +642,7 @@ class GenericUpdater(object): """ -@six.add_metaclass(abc.ABCMeta) -class RenewDeployer(object): +class RenewDeployer(object, metaclass=abc.ABCMeta): """Interface for update types run when a lineage is renewed This class allows plugins to perform types of updates that need to run at diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 75ce9e2ff..dcbf0381a 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -36,7 +36,7 @@ except (ImportError, AttributeError): # pragma: no cover logger = logging.getLogger(__name__) -class RevocationChecker(object): +class RevocationChecker: """This class figures out OCSP checking on this system, and performs it.""" def __init__(self, enforce_openssl_binary_usage=False): @@ -167,7 +167,7 @@ def _determine_ocsp_server(cert_path): if host: return url, host - logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) + logger.info("Cannot process OCSP host from URL (%s) in certificate at %s", url, cert_path) return None, None @@ -222,7 +222,7 @@ def _check_ocsp_cryptography(cert_path, chain_path, url, timeout): def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert, cert_path): - """Verify that the OCSP is valid for serveral criteria""" + """Verify that the OCSP is valid for several criteria""" # Assert OCSP response corresponds to the certificate we are talking about if response_ocsp.serial_number != request_ocsp.serial_number: raise AssertionError('the certificate in response does not correspond ' diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index d3fb5b7dc..489c75a3d 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -40,7 +40,7 @@ hostname_regex = re.compile( @zope.interface.implementer(interfaces.IPlugin) -class Plugin(object): +class Plugin: """Generic plugin.""" # provider is not inherited, subclasses must define it on their own # @zope.interface.provider(interfaces.IPluginFactory) @@ -201,7 +201,7 @@ class Installer(Plugin): constants.ALL_SSL_DHPARAMS_HASHES) -class Addr(object): +class Addr: r"""Represents an virtual host address. :param str addr: addr part of vhost address @@ -299,7 +299,7 @@ class Addr(object): return result -class ChallengePerformer(object): +class ChallengePerformer: """Abstract base for challenge performers. :ivar configurator: Authenticator and installer plugin diff --git a/certbot/certbot/plugins/dns_common.py b/certbot/certbot/plugins/dns_common.py index 245b7dc05..5c0fbcba9 100644 --- a/certbot/certbot/plugins/dns_common.py +++ b/certbot/certbot/plugins/dns_common.py @@ -233,7 +233,7 @@ class DNSAuthenticator(common.Plugin): raise errors.PluginError('{0} required to proceed.'.format(label)) -class CredentialsConfiguration(object): +class CredentialsConfiguration: """Represents a user-supplied filed which stores API credentials.""" def __init__(self, filename, mapper=lambda x: x): diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index c3d80ca29..a29509b79 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -23,7 +23,7 @@ except ImportError: logger = logging.getLogger(__name__) -class LexiconClient(object): +class LexiconClient: """ Encapsulates all communication with a DNS provider via Lexicon. """ diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index d5044d336..1affcae4e 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -6,7 +6,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock # type: ignore -import six from acme import challenges from certbot import achallenges @@ -18,7 +17,7 @@ DOMAIN = 'example.com' KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) -class BaseAuthenticatorTest(object): +class BaseAuthenticatorTest: """ A base test class to reduce duplication between test code for DNS Authenticator Plugins. @@ -31,7 +30,7 @@ class BaseAuthenticatorTest(object): challb=acme_util.DNS01, domain=DOMAIN, account_key=KEY) def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) # pylint: disable=no-member + self.assertTrue(isinstance(self.auth.more_info(), str)) # pylint: disable=no-member def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref(None), [challenges.DNS01]) # pylint: disable=no-member diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index 1bef06042..adeb1a268 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -34,7 +34,7 @@ class BaseLexiconAuthenticatorTest(dns_test_common.BaseAuthenticatorTest): self.assertEqual(expected, self.mock_client.mock_calls) -class BaseLexiconClientTest(object): +class BaseLexiconClientTest: DOMAIN_NOT_FOUND = Exception('No domain found') GENERIC_ERROR = RequestException LOGIN_ERROR = HTTPError('400 Client Error: ...') diff --git a/certbot/certbot/plugins/enhancements.py b/certbot/certbot/plugins/enhancements.py index 4abce2d2f..e674c32a2 100644 --- a/certbot/certbot/plugins/enhancements.py +++ b/certbot/certbot/plugins/enhancements.py @@ -1,8 +1,6 @@ """New interface style Certbot enhancements""" import abc -import six - from acme.magic_typing import Any from acme.magic_typing import Dict from acme.magic_typing import List @@ -91,8 +89,7 @@ def populate_cli(add): help=enh["cli_help"]) -@six.add_metaclass(abc.ABCMeta) -class AutoHSTSEnhancement(object): +class AutoHSTSEnhancement(object, metaclass=abc.ABCMeta): """ Enhancement interface that installer plugins can implement in order to provide functionality that configures the software to have a diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index f3ed14dce..abef534f9 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -11,7 +11,7 @@ from certbot.compat import os logger = logging.getLogger(__name__) -class PluginStorage(object): +class PluginStorage: """Class implementing storage functionality for plugins""" def __init__(self, config, classkey): diff --git a/certbot/certbot/plugins/util.py b/certbot/certbot/plugins/util.py index 87eb45fe9..04a741da6 100644 --- a/certbot/certbot/plugins/util.py +++ b/certbot/certbot/plugins/util.py @@ -10,9 +10,11 @@ logger = logging.getLogger(__name__) def get_prefixes(path): """Retrieves all possible path prefixes of a path, in descending order - of length. For instance, - (linux) /a/b/c returns ['/a/b/c', '/a/b', '/a', '/'] - (windows) C:\\a\\b\\c returns ['C:\\a\\b\\c', 'C:\\a\\b', 'C:\\a', 'C:'] + of length. For instance: + + * (Linux) `/a/b/c` returns `['/a/b/c', '/a/b', '/a', '/']` + * (Windows) `C:\\a\\b\\c` returns `['C:\\a\\b\\c', 'C:\\a\\b', 'C:\\a', 'C:']` + :param str path: the path to break into prefixes :returns: all possible path prefixes of given path in descending order diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index 58e1216b7..e6777a7da 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -3,11 +3,9 @@ import csv import glob import logging import shutil -import sys import time import traceback -import six from certbot import errors from certbot import util @@ -18,7 +16,7 @@ from certbot.compat import os logger = logging.getLogger(__name__) -class Reverter(object): +class Reverter: """Reverter Class - save and revert configuration checkpoints. This class can be used by the plugins, especially Installers, to @@ -252,11 +250,10 @@ class Reverter(object): def _run_undo_commands(self, filepath): """Run all commands in a file.""" - # NOTE: csv module uses native strings. That is, bytes on Python 2 and - # unicode on Python 3 + # NOTE: csv module uses native strings. That is unicode on Python 3 # It is strongly advised to set newline = '' on Python 3 with CSV, # and it fixes problems on Windows. - kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + kwargs = {'newline': ''} with open(filepath, 'r', **kwargs) as csvfile: # type: ignore csvreader = csv.reader(csvfile) for command in reversed(list(csvreader)): @@ -355,7 +352,7 @@ class Reverter(object): command_file = None # It is strongly advised to set newline = '' on Python 3 with CSV, # and it fixes problems on Windows. - kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + kwargs = {'newline': ''} try: if os.path.isfile(commands_fp): command_file = open(commands_fp, "a", **kwargs) # type: ignore @@ -518,7 +515,7 @@ class Reverter(object): # It is possible save checkpoints faster than 1 per second resulting in # collisions in the naming convention. - for _ in six.moves.range(2): + for _ in range(2): timestamp = self._checkpoint_timestamp() final_dir = os.path.join(self.config.backup_dir, timestamp) try: diff --git a/certbot/certbot/tests/acme_util.py b/certbot/certbot/tests/acme_util.py index f4a20ea86..49ca88bf5 100644 --- a/certbot/certbot/tests/acme_util.py +++ b/certbot/certbot/tests/acme_util.py @@ -2,7 +2,6 @@ import datetime import josepy as jose -import six from acme import challenges from acme import messages @@ -69,7 +68,7 @@ def gen_authzr(authz_status, domain, challs, statuses, combos=True): """ challbs = tuple( chall_to_challb(chall, status) - for chall, status in six.moves.zip(challs, statuses) + for chall, status in zip(challs, statuses) ) authz_kwargs = { "identifier": messages.Identifier( diff --git a/certbot/certbot/tests/testdata/README b/certbot/certbot/tests/testdata/README index 867215916..ade2fae2b 100644 --- a/certbot/certbot/tests/testdata/README +++ b/certbot/certbot/tests/testdata/README @@ -2,10 +2,16 @@ The following command has been used to generate test keys: for x in 256 512 2048; do openssl genrsa -out rsa${k}_key.pem $k; done +For the elliptic curve private keys, this command was used: + + for k in "prime256v1" "secp384r1" "secp521r1" do + openssl genpkey -algorithm ${k} -out ec_${k}_key.pem + done + and for the CSR PEM (Certificate Signing Request): openssl req -new -out csr-Xsans_X.pem -key rsa512_key.pem [-config csr-Xsans_X.conf | -subj '/CN=example.com'] [-outform DER > csr_X.der] and for the certificate: - openssl req -new -out cert_X.pem -key rsaX_key.pem -subj '/CN=example.com' -x509 [-outform DER > cert_X.der] \ No newline at end of file + openssl req -new -out cert_X.pem -key rsaX_key.pem -subj '/CN=example.com' -x509 [-outform DER > cert_X.der] diff --git a/certbot/certbot/tests/testdata/ec_prime256v1_key.pem b/certbot/certbot/tests/testdata/ec_prime256v1_key.pem new file mode 100644 index 000000000..77552a656 --- /dev/null +++ b/certbot/certbot/tests/testdata/ec_prime256v1_key.pem @@ -0,0 +1,8 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIDqQPQl69kuh+DrecC8SFPt21f0F/HHDP3T4/Lf0zIVFoAoGCCqGSM49 +AwEHoUQDQgAEHou50Ee9u+8Vial6VbUHExlzsiCHtORlW0X0pKo5RspIKB0QyKwo +dUXvBbv95I9yCO5+MlGkKjwLHtIEze0Hww== +-----END EC PRIVATE KEY----- diff --git a/certbot/certbot/tests/testdata/ec_secp384r1_key.pem b/certbot/certbot/tests/testdata/ec_secp384r1_key.pem new file mode 100644 index 000000000..19a846a46 --- /dev/null +++ b/certbot/certbot/tests/testdata/ec_secp384r1_key.pem @@ -0,0 +1,9 @@ +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDAgvZGw5C7Mp26N0cXA+vIg5K/J5MJw+MVGYfGF4ZutuCLeYMrWT68R +A0h6hJvDtMSgBwYFK4EEACKhZANiAAR1uQYZeU5Kml5o53Q8/PCdwUbqdgCSkV0C +J5a6bhDRMp20fdp2T/mbkdxuVEl81lqfKPZhsd4CZsLaVIU3RUoGgIT1R3QKawpJ +SuXq37yWFX2hqlgt+lsBufZ8RD5QnZc= +-----END EC PRIVATE KEY----- diff --git a/certbot/certbot/tests/testdata/ec_secp521r1_key.pem b/certbot/certbot/tests/testdata/ec_secp521r1_key.pem new file mode 100644 index 000000000..78bb9a0ea --- /dev/null +++ b/certbot/certbot/tests/testdata/ec_secp521r1_key.pem @@ -0,0 +1,10 @@ +-----BEGIN EC PARAMETERS----- +BgUrgQQAIw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIHcAgEBBEIACWWVKm1qAIejZ6qmqk9D69wQW5FAe3Er0IxWAMkonTEhu8EH5Q2i +2vT2bESm730zhGTe2Pn11b85H6UI9hxhCHygBwYFK4EEACOhgYkDgYYABAEQi1WF +m3suHjPyWACyOJYGUn1Kx6rfBo0PjC7X2TU9jr8umLkIpaaF5UsBuMBmdz1IHL0U +k0gQtoOQ0Qu8N74GuAGzGR0S3RYIv6gfYVz3dS1K4n4b307Lx62bnvtlNxcIvt3w +hmS5OdvQ1Kdxh6oqbSVhhbQmJcgab78Txx3R2QeCxw== +-----END EC PRIVATE KEY----- diff --git a/certbot/certbot/tests/testdata/sample-archive-ec/cert1.pem b/certbot/certbot/tests/testdata/sample-archive-ec/cert1.pem new file mode 100644 index 000000000..b07af2bf7 --- /dev/null +++ b/certbot/certbot/tests/testdata/sample-archive-ec/cert1.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC2zCCAcOgAwIBAgIIBvrEnbPRYu8wDQYJKoZIhvcNAQELBQAwKDEmMCQGA1UE +AxMdUGViYmxlIEludGVybWVkaWF0ZSBDQSAxMjZjNGIwHhcNMjAxMDEyMjEwNzQw +WhcNMjUxMDEyMjEwNzQwWjAjMSEwHwYDVQQDExhjLmVuY3J5cHRpb24tZXhhbXBs +ZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARjMhuW0ENPPC33PjB5XsYU +CRw640kPQENIDatcTJaENZIZdqKd6rI6jc+lpbmXot7Zi52clJlSJS+V6oDAt2Lh +o4HYMIHVMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB +BQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUj7Kd3ENqxlPf8B2bIGhsjydX +mPswHwYDVR0jBBgwFoAUEiGxlkRsi+VvcogH5dVD3h1laAcwMQYIKwYBBQUHAQEE +JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vMTI3LjAuMC4xOjQwMDIwIwYDVR0RBBww +GoIYYy5lbmNyeXB0aW9uLWV4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQCl +k0JXsa8y7fg41WWMDhw60bPW77O0FtOmTcnhdI5daYNemQVk+Q5EMaBLQ/oGjgXd +9QXFzXH1PL904YEnSLt+iTpXn++7rQSNzQsdYqw0neWk4f5pEBiN+WORpb6mwobV +ifMtBOkNEHvrJ2Pkci9U1lLwtKD/DSew6QtJU5DSkmH1XdGuMJiubygEIvELtvgq +cP9S368ZvPmPGmKaJQXBiuaR8MTjY/Bkr79aXQMjKbf+mpn7h0POCcePk1DY/rm6 +Da+X16lf0hHyQhSUa7Vgyim6rK1/hlw+Z00i+sQCKD9Ih7kXuuGqfSDC33cfO8Tj +o/MXO8lcxkrem5zU5QWP +-----END CERTIFICATE----- diff --git a/certbot/certbot/tests/testdata/sample-archive-ec/chain1.pem b/certbot/certbot/tests/testdata/sample-archive-ec/chain1.pem new file mode 100644 index 000000000..64a805c0f --- /dev/null +++ b/certbot/certbot/tests/testdata/sample-archive-ec/chain1.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDUDCCAjigAwIBAgIIbi787yVrcMAwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE +AxMVUGViYmxlIFJvb3QgQ0EgMGM1MjI1MCAXDTIwMTAxMjIwMjI0NloYDzIwNTAx +MDEyMjEyMjQ2WjAoMSYwJAYDVQQDEx1QZWJibGUgSW50ZXJtZWRpYXRlIENBIDEy +NmM0YjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALGeVk1BMJraeqRq +mJ2+hgso8VOAv2s2CVxUJjIVcn7f2adE8NyTsSQ1brlsnKCUYUw7yLTQH0izLQRB +qKVIDFkUqo5/FuTJ2QlfA2EwBL8J7s/7L7vj3L0DiVpwgxPSyFEwdl/Y5y7ofsX5 +CIhCFcaMAmTIuKLiSfCJjGwkbEMuolm+lO8Mikxxc/JtDVUC479ugU7PU9O09bMH +nm+sD6Bgd+KMoPkCCCoeShJS9X3Ziq9HGc7Z6nhM/zirFARt2XkonEdAZ8br01zY +MRiY9txhlWQ7mUkOtzOSoEuYJNoUbvMUf0+tNzto26WRyF7dJmh7lTBsYrvAwUTx +PzNyst0CAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYB +BQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBIhsZZE +bIvlb3KIB+XVQ94dZWgHMB8GA1UdIwQYMBaAFOaKTaXg37vKgRt7d79YOjAoAtJT +MA0GCSqGSIb3DQEBCwUAA4IBAQAU2mZii7PH2pkw2lNM0QqPbcW/UYyvFoUeM8Aq +uCtsI2s+oxCJTqzfLsA0N8NY4nHLQ5wAlNJfJekngni8hbmJTKU4JFTMe7kLQO8P +fJbk0pTzhhHVQw7CVwB6Pwq3u2m/JV+d6xDIDc+AVkuEl19ZJU0rTWyooClfFLZV +EdZmEiUtA3PGlxoYwYhoGHYlhFxsoFONhCsBEdN7k7FKtFGVxN7oc5SKmKp0YZTW +fcrEtrdNThATO4ymhCC2zh33NI/MT1O74fpaAc2k6LcTl57MKiLfTYX4LTL6v9JG +9tlNqjFVRRmzEbtXTPcCb+w9g1VqoOGok7mGXYLTYtShCuvE +-----END CERTIFICATE----- diff --git a/certbot/certbot/tests/testdata/sample-archive-ec/fullchain1.pem b/certbot/certbot/tests/testdata/sample-archive-ec/fullchain1.pem new file mode 100644 index 000000000..1ba80ba4e --- /dev/null +++ b/certbot/certbot/tests/testdata/sample-archive-ec/fullchain1.pem @@ -0,0 +1,38 @@ +-----BEGIN CERTIFICATE----- +MIIC2zCCAcOgAwIBAgIILlmGtZhUFEwwDQYJKoZIhvcNAQELBQAwKDEmMCQGA1UE +AxMdUGViYmxlIEludGVybWVkaWF0ZSBDQSAxMjZjNGIwHhcNMjAxMDEyMjA1MDM0 +WhcNMjUxMDEyMjA1MDM0WjAjMSEwHwYDVQQDExhjLmVuY3J5cHRpb24tZXhhbXBs +ZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARHEzR8JPWrEmpmgM+F2bk5 +9mT0u6CjzmJG0QpbaqprLiG5NGpW84VQ5TFCrmC4KxYfigCfMhfHRNfFYvNUK3V/ +o4HYMIHVMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB +BQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU1CsVL+bPnzaxxQ5jUENmQJIO +lKwwHwYDVR0jBBgwFoAUEiGxlkRsi+VvcogH5dVD3h1laAcwMQYIKwYBBQUHAQEE +JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vMTI3LjAuMC4xOjQwMDIwIwYDVR0RBBww +GoIYYy5lbmNyeXB0aW9uLWV4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQBn +2D8loC7pfk28JYpFLr5lmFKJWWmtLGlpsWDj61fVjtTfGKLziJz+MM6il4Y3hIz5 +58qiFK0ue0M63dIBJ33N+XxSEXon4Q0gy/zRWfH9jtPJ3FwfjkU/RT9PAUClYi0G +ptNWnTmgQkNzousbcAtRNXuuShH3856vhUnwkX+xM+cbIDi1JVmFjcGrEEQJ0rUF +mv2ZTyfbWbUs3v4rReETi2NVzr1Ql6J+ByNcMvHODzFy3t0L6yelAw2ca1I+c9HU ++Z0tnp/ykR7eXNuVLivok8UBf5OC413lh8ZO5g+Bgzh/LdtkUuavg1MYtEX0H6mX +9U7y3nVI8WEbPGf+HDeu +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDUDCCAjigAwIBAgIIbi787yVrcMAwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE +AxMVUGViYmxlIFJvb3QgQ0EgMGM1MjI1MCAXDTIwMTAxMjIwMjI0NloYDzIwNTAx +MDEyMjEyMjQ2WjAoMSYwJAYDVQQDEx1QZWJibGUgSW50ZXJtZWRpYXRlIENBIDEy +NmM0YjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALGeVk1BMJraeqRq +mJ2+hgso8VOAv2s2CVxUJjIVcn7f2adE8NyTsSQ1brlsnKCUYUw7yLTQH0izLQRB +qKVIDFkUqo5/FuTJ2QlfA2EwBL8J7s/7L7vj3L0DiVpwgxPSyFEwdl/Y5y7ofsX5 +CIhCFcaMAmTIuKLiSfCJjGwkbEMuolm+lO8Mikxxc/JtDVUC479ugU7PU9O09bMH +nm+sD6Bgd+KMoPkCCCoeShJS9X3Ziq9HGc7Z6nhM/zirFARt2XkonEdAZ8br01zY +MRiY9txhlWQ7mUkOtzOSoEuYJNoUbvMUf0+tNzto26WRyF7dJmh7lTBsYrvAwUTx +PzNyst0CAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYB +BQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBIhsZZE +bIvlb3KIB+XVQ94dZWgHMB8GA1UdIwQYMBaAFOaKTaXg37vKgRt7d79YOjAoAtJT +MA0GCSqGSIb3DQEBCwUAA4IBAQAU2mZii7PH2pkw2lNM0QqPbcW/UYyvFoUeM8Aq +uCtsI2s+oxCJTqzfLsA0N8NY4nHLQ5wAlNJfJekngni8hbmJTKU4JFTMe7kLQO8P +fJbk0pTzhhHVQw7CVwB6Pwq3u2m/JV+d6xDIDc+AVkuEl19ZJU0rTWyooClfFLZV +EdZmEiUtA3PGlxoYwYhoGHYlhFxsoFONhCsBEdN7k7FKtFGVxN7oc5SKmKp0YZTW +fcrEtrdNThATO4ymhCC2zh33NI/MT1O74fpaAc2k6LcTl57MKiLfTYX4LTL6v9JG +9tlNqjFVRRmzEbtXTPcCb+w9g1VqoOGok7mGXYLTYtShCuvE +-----END CERTIFICATE----- diff --git a/certbot/certbot/tests/testdata/sample-archive-ec/privkey1.pem b/certbot/certbot/tests/testdata/sample-archive-ec/privkey1.pem new file mode 100644 index 000000000..6843b83d6 --- /dev/null +++ b/certbot/certbot/tests/testdata/sample-archive-ec/privkey1.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgNgefv2dad4U1VYEi +0WkdHuqywi5QXAe30OwNTTGjhbihRANCAARHEzR8JPWrEmpmgM+F2bk59mT0u6Cj +zmJG0QpbaqprLiG5NGpW84VQ5TFCrmC4KxYfigCfMhfHRNfFYvNUK3V/ +-----END PRIVATE KEY----- diff --git a/certbot/certbot/tests/testdata/sample-renewal-deprecated-option.conf b/certbot/certbot/tests/testdata/sample-renewal-deprecated-option.conf new file mode 100644 index 000000000..2b777d3de --- /dev/null +++ b/certbot/certbot/tests/testdata/sample-renewal-deprecated-option.conf @@ -0,0 +1,14 @@ +# renew_before_expiry = 30 days +version = 1.11.0 +archive_dir = MAGICDIR/live/sample-renewal-deprecated-option +cert = MAGICDIR/live/sample-renewal-deprecated-option/cert.pem +privkey = MAGICDIR/live/sample-renewal-deprecated-option/privkey.pem +chain = MAGICDIR/live/sample-renewal-deprecated-option/chain.pem +fullchain = MAGICDIR/live/sample-renewal-deprecated-option/fullchain.pem + +# Options used in the renewal process +[renewalparams] +account = ffffffffffffffffffffffffffffffff +authenticator = nginx +installer = nginx +manual_public_ip_logging_ok = None diff --git a/certbot/certbot/tests/testdata/sample-renewal-ec.conf b/certbot/certbot/tests/testdata/sample-renewal-ec.conf new file mode 100644 index 000000000..d8a4b32bf --- /dev/null +++ b/certbot/certbot/tests/testdata/sample-renewal-ec.conf @@ -0,0 +1,79 @@ +# add some stuff here +# assets/integration_tests + +cert = MAGICDIR/live/sample-renewal-ec/cert.pem +privkey = MAGICDIR/live/sample-renewal-ec/privkey.pem +chain = MAGICDIR/live/sample-renewal-ec/chain.pem +fullchain = MAGICDIR/live/sample-renewal-ec/fullchain.pem +renew_before_expiry = 4 years + +# Options and defaults used in the renewal process +[renewalparams] +no_self_upgrade = False +apache_enmod = a2enmod +no_verify_ssl = False +ifaces = None +apache_dismod = a2dismod +register_unsafely_without_email = False +apache_handle_modules = True +uir = None +installer = None +nginx_ctl = nginx +config_dir = MAGICDIR +text_mode = False +func = +staging = True +prepare = False +work_dir = /var/lib/letsencrypt +tos = False +init = False +http01_port = 80 +duplicate = False +noninteractive_mode = True +key_path = None +nginx = False +nginx_server_root = /etc/nginx +fullchain_path = /home/ubuntu/letsencrypt/chain.pem +email = None +csr = None +agree_dev_preview = None +redirect = None +verb = certonly +verbose_count = -3 +config_file = None +renew_by_default = False +hsts = False +apache_handle_sites = True +authenticator = standalone +domains = isnot.org, +key_type = ecdsa +elliptic_curve = secp256r1 +apache_challenge_location = /etc/apache2 +checkpoints = 1 +manual_test_mode = False +apache = False +cert_path = /home/ubuntu/letsencrypt/cert.pem +webroot_path = None +reinstall = False +expand = False +strict_permissions = False +apache_server_root = /etc/apache2 +account = None +dry_run = False +manual_public_ip_logging_ok = False +chain_path = /home/ubuntu/letsencrypt/chain.pem +break_my_certs = False +standalone = True +manual = False +server = https://acme-staging-v02.api.letsencrypt.org/directory +webroot = False +os_packages_only = False +apache_init_script = None +user_agent = None +apache_le_vhost_ext = -le-ssl.conf +debug = False +logs_dir = /var/log/letsencrypt +apache_vhost_root = /etc/apache2/sites-available +configurator = None +must_staple = True +[[webroot_map]] diff --git a/certbot/certbot/tests/testdata/sample-renewal.conf b/certbot/certbot/tests/testdata/sample-renewal.conf index 936c5c0e0..0c56781b3 100644 --- a/certbot/certbot/tests/testdata/sample-renewal.conf +++ b/certbot/certbot/tests/testdata/sample-renewal.conf @@ -44,6 +44,7 @@ apache_handle_sites = True authenticator = standalone domains = isnot.org, rsa_key_size = 2048 +elliptic_curve = secp256r1 apache_challenge_location = /etc/apache2 checkpoints = 1 manual_test_mode = False diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 92f52a852..78236fa06 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -1,4 +1,6 @@ """Test utilities.""" +from importlib import reload as reload_module +import io import logging from multiprocessing import Event from multiprocessing import Process @@ -6,18 +8,13 @@ import shutil import sys import tempfile import unittest +import warnings from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore import OpenSSL import pkg_resources -import six -from six.moves import reload_module from certbot import interfaces from certbot import util @@ -29,6 +26,18 @@ from certbot.compat import filesystem from certbot.compat import os from certbot.display import util as display_util +try: + import mock + warnings.warn( + "The external mock module is being used for backwards compatibility " + "since it is available, however, future versions of Certbot's tests will " + "use unittest.mock. Be sure to update your code accordingly.", + PendingDeprecationWarning + ) +except ImportError: # pragma: no cover + from unittest import mock # type: ignore + + def vector_path(*names): """Path to a test vector.""" @@ -93,7 +102,7 @@ def load_pyopenssl_private_key(*names): return OpenSSL.crypto.load_privatekey(loader, load_vector(*names)) -def make_lineage(config_dir, testfile): +def make_lineage(config_dir, testfile, ec=False): """Creates a lineage defined by testfile. This creates the archive, live, and renewal directories if @@ -119,7 +128,7 @@ def make_lineage(config_dir, testfile): if not os.path.exists(directory): filesystem.makedirs(directory) - sample_archive = vector_path('sample-archive') + sample_archive = vector_path('sample-archive{}'.format('-ec' if ec else '')) for kind in os.listdir(sample_archive): shutil.copyfile(os.path.join(sample_archive, kind), os.path.join(archive_dir, kind)) @@ -170,13 +179,13 @@ def patch_get_utility_with_stdout(target='zope.component.getUtility', :rtype: mock.MagicMock """ - stdout = stdout if stdout else six.StringIO() + stdout = stdout if stdout else io.StringIO() freezable_mock = _create_get_utility_mock_with_stdout(stdout) return mock.patch(target, new=freezable_mock) -class FreezableMock(object): +class FreezableMock: """Mock object with the ability to freeze attributes. This class works like a regular mock.MagicMock object, except diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index fd7b46f85..ffd3a65f2 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -4,7 +4,6 @@ import argparse import atexit import collections -from collections import OrderedDict import distutils.version import errno import logging @@ -15,8 +14,8 @@ import subprocess import sys import configargparse -import six +from acme.magic_typing import Dict from acme.magic_typing import Text from acme.magic_typing import Tuple from acme.magic_typing import Union @@ -59,7 +58,7 @@ _INITIAL_PID = os.getpid() # the dict are attempted to be cleaned up at program exit. If the # program exits before the lock is cleaned up, it is automatically # released, but the file isn't deleted. -_LOCKS = OrderedDict() # type: OrderedDict[str, lock.LockFile] +_LOCKS = {} # type: Dict[str, lock.LockFile] def env_no_snap_for_external_calls(): @@ -153,7 +152,7 @@ def lock_dir_until_exit(dir_path): def _release_locks(): - for dir_lock in six.itervalues(_LOCKS): + for dir_lock in _LOCKS.values(): try: dir_lock.release() except: # pylint: disable=bare-except @@ -439,7 +438,7 @@ def safe_email(email): return False -class _ShowWarning(argparse.Action): +class DeprecatedArgumentAction(argparse.Action): """Action to log a warning when an argument is used.""" def __call__(self, unused1, unused2, unused3, option_string=None): logger.warning("Use of %s is deprecated.", option_string) @@ -458,16 +457,16 @@ def add_deprecated_argument(add_argument, argument_name, nargs): :param nargs: Value for nargs when adding the argument to argparse. """ - if _ShowWarning not in configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE: + if DeprecatedArgumentAction not in configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE: # In version 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE was # changed from a set to a tuple. if isinstance(configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE, set): configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE.add( - _ShowWarning) + DeprecatedArgumentAction) else: configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE += ( - _ShowWarning,) - add_argument(argument_name, action=_ShowWarning, + DeprecatedArgumentAction,) + add_argument(argument_name, action=DeprecatedArgumentAction, help=argparse.SUPPRESS, nargs=nargs) @@ -517,7 +516,7 @@ def enforce_domain_sanity(domain): """ # Unicode try: - if isinstance(domain, six.binary_type): + if isinstance(domain, bytes): domain = domain.decode('utf-8') domain.encode('ascii') except UnicodeError: @@ -579,7 +578,7 @@ def is_wildcard_domain(domain): """ wildcard_marker = b"*." # type: Union[Text, bytes] - if isinstance(domain, six.text_type): + if isinstance(domain, str): wildcard_marker = u"*." return domain.startswith(wildcard_marker) diff --git a/certbot/docs/api/certbot.compat.os.rst b/certbot/docs/api/certbot.compat.os.rst index 3a4c9fe47..c46cee668 100644 --- a/certbot/docs/api/certbot.compat.os.rst +++ b/certbot/docs/api/certbot.compat.os.rst @@ -2,6 +2,4 @@ certbot.compat.os module ======================== .. automodule:: certbot.compat.os - :members: - :undoc-members: - :show-inheritance: + :members: chmod, umask, chown, open, mkdir, makedirs, rename, replace, access, stat, fstat diff --git a/certbot/docs/ciphers.rst b/certbot/docs/ciphers.rst index 294e6a7fa..43f648898 100644 --- a/certbot/docs/ciphers.rst +++ b/certbot/docs/ciphers.rst @@ -80,8 +80,8 @@ its preferences in accordance with its own policy or its administrators' preferences, and use different cryptographic mechanisms or parameters, or a different priority order, than the defaults provided by Certbot. -If you don't use Certbot to configure your server directly, because the -client doesn't integrate with your server software or because you chose +If you don't use Certbot to configure your server directly, because the +client doesn't integrate with your server software or because you chose not to use this integration, then the cryptographic defaults haven't been modified, and the cryptography chosen by the server will still be whatever the default for your software was. For example, if you obtain a @@ -254,7 +254,7 @@ https://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/elb-secur U.S. Government 18F ~~~~~~~~~~~~~~~~~~~ -The 18F site (https://18f.gsa.gov/) is using +The 18F site (https://18f.gsa.gov/) is using :: diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index 3c3497282..27511777b 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -99,9 +99,9 @@ optional arguments: before submitting to CA (default: False) --preferred-chain PREFERRED_CHAIN If the CA offers multiple certificate chains, prefer - the chain with an issuer matching this Subject Common - Name. If no match, the default offered chain will be - used. (default: None) + the chain whose topmost certificate was issued from + this Subject Common Name. If no match, the default + offered chain will be used. (default: None) --preferred-challenges PREF_CHALLS A sorted, comma delimited list of the preferred challenge to use during authorization with the most @@ -118,12 +118,12 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.9.0 (certbot(-auto); - OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). - The flags encoded in the user agent are: --duplicate, - --force-renew, --allow-subset-of-names, -n, and - whether any hooks are set. + "". (default: CertbotACMEClient/1.13.0 + (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX + Installer/YYY (SUBCOMMAND; flags: FLAGS) + Py/major.minor.patchlevel). The flags encoded in the + user agent are: --duplicate, --force-renew, --allow- + subset-of-names, -n, and whether any hooks are set. --user-agent-comment USER_AGENT_COMMENT Add a comment to the default user agent string. May be used when repackaging Certbot or calling it from @@ -167,19 +167,6 @@ automation: --duplicate Allow making a certificate lineage that duplicates an existing one (both can be renewed in parallel) (default: False) - --os-packages-only (certbot-auto only) install OS package dependencies - and then stop (default: False) - --no-self-upgrade (certbot-auto only) prevent the certbot-auto script - from upgrading itself to newer released versions - (default: Upgrade automatically) - --no-bootstrap (certbot-auto only) prevent the certbot-auto script - from installing OS-level dependencies (default: Prompt - to install OS-wide dependencies, but exit if the user - says 'No') - --no-permissions-check - (certbot-auto only) skip the check on the file system - permissions of the certbot-auto script (default: - False) -q, --quiet Silence all output except errors. Useful for automation via cron. Implies --non-interactive. (default: False) @@ -188,6 +175,12 @@ security: Security parameters & server settings --rsa-key-size N Size of the RSA key. (default: 2048) + --key-type {rsa,ecdsa} + Type of generated private key(Only *ONE* per + invocation can be provided at this time) (default: + rsa) + --elliptic-curve N The SECG elliptic curve name to use. Please see RFC + 8446 for supported values. (default: secp256r1) --must-staple Adds the OCSP Must Staple extension to the certificate. Autoconfigures OCSP Stapling for supported setups (Apache version >= 2.3.3 ). (default: @@ -248,8 +241,8 @@ paths: Flags for changing execution paths & servers --cert-path CERT_PATH - Path to where certificate is saved (with auth --csr), - installed from, or revoked. (default: None) + Path to where certificate is saved (with certonly + --csr), installed from, or revoked (default: None) --key-path KEY_PATH Path to private key for certificate installation or revocation (if account key is missing) (default: None) --fullchain-path FULLCHAIN_PATH @@ -533,8 +526,8 @@ dns-cloudxns: CloudXNS credentials INI file. (default: None) dns-digitalocean: - Obtain certs using a DNS TXT record (if you are using DigitalOcean for - DNS). + Obtain certificates using a DNS TXT record (if you are using DigitalOcean + for DNS). --dns-digitalocean-propagation-seconds DNS_DIGITALOCEAN_PROPAGATION_SECONDS The number of seconds to wait for DNS to propagate @@ -595,7 +588,8 @@ dns-google: therequired permissions.) (default: None) dns-linode: - Obtain certs using a DNS TXT record (if you are using Linode for DNS). + Obtain certificates using a DNS TXT record (if you are using Linode for + DNS). --dns-linode-propagation-seconds DNS_LINODE_PROPAGATION_SECONDS The number of seconds to wait for DNS to propagate @@ -688,8 +682,6 @@ manual: --manual-cleanup-hook MANUAL_CLEANUP_HOOK Path or command to execute for the cleanup script (default: None) - --manual-public-ip-logging-ok - Automatically allows public IP logging (default: Ask) nginx: Nginx Web Server plugin diff --git a/certbot/docs/compatibility.rst b/certbot/docs/compatibility.rst index a511f36a2..a4f33c281 100644 --- a/certbot/docs/compatibility.rst +++ b/certbot/docs/compatibility.rst @@ -9,7 +9,7 @@ application itself. This means that we will not change behavior in a backwards incompatible way except in a new major version of the project. .. note:: None of this applies to the behavior of Certbot distribution - mechanisms such as :ref:`certbot-auto ` or OS packages whose + mechanisms such as :ref:`our snaps ` or OS packages whose behavior may change at any time. Semantic versioning only applies to the common Certbot components that are installed by various distribution methods. diff --git a/certbot/docs/conf.py b/certbot/docs/conf.py index dbd4067d5..254bd3edd 100644 --- a/certbot/docs/conf.py +++ b/certbot/docs/conf.py @@ -95,7 +95,11 @@ language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = [ + '_build', + 'challenges.rst', + 'ciphers.rst' +] # The reST default role (used for this markup: `text`) to use for all # documents. diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 32a1ee72b..def2c7fcd 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -56,18 +56,18 @@ Set up the Python virtual environment that will host your Certbot local instance .. code-block:: shell cd certbot - python tools/venv3.py + python tools/venv.py .. note:: You may need to repeat this when Certbot's dependencies change or when a new plugin is introduced. You can now run the copy of Certbot from git either by executing -``venv3/bin/certbot``, or by activating the virtual environment. You can do the +``venv/bin/certbot``, or by activating the virtual environment. You can do the latter by running: .. code-block:: shell - source venv3/bin/activate + source venv/bin/activate After running this command, ``certbot`` and development tools like ``ipdb``, ``ipython``, ``pytest``, and ``tox`` are available in the shell where you ran @@ -169,7 +169,7 @@ To do so you need: - Docker installed, and a user with access to the Docker client, - an available `local copy`_ of Certbot. -The virtual environment set up with `python tools/venv3.py` contains two commands +The virtual environment set up with `python tools/venv.py` contains two CLI tools that can be used once the virtual environment is activated: .. code-block:: shell @@ -180,6 +180,9 @@ that can be used once the virtual environment is activated: - Press CTRL+C to stop this instance. - This instance is configured to validate challenges against certbot executed locally. +.. note:: Some options are available to tweak the local ACME server. You can execute + ``run_acme_server --help`` to see the inline help of the ``run_acme_server`` tool. + .. code-block:: shell certbot_test [ARGS...] @@ -194,8 +197,8 @@ using an HTTP-01 challenge on a machine with Python 3: .. code-block:: shell - python tools/venv3.py - source venv3/bin/activate + python tools/venv.py + source venv/bin/activate run_acme_server & certbot_test certonly --standalone -d test.example.com # To stop Pebble, launch `fg` to get back the background job, then press CTRL+C @@ -279,8 +282,8 @@ support for IIS, Icecast and Plesk. Installers and Authenticators will oftentimes be the same class/object (because for instance both tasks can be performed by a webserver like nginx) though this is not always the case (the standalone plugin is an authenticator -that listens on port 80, but it cannot install certs; a postfix plugin would -be an installer but not an authenticator). +that listens on port 80, but it cannot install certificates; a postfix plugin +would be an installer but not an authenticator). Installers and Authenticators are kept separate because it should be possible to use the `~.StandaloneAuthenticator` (it sets @@ -431,16 +434,23 @@ Please: 4. Remember to use ``pylint``. +5. You may consider installing a plugin for `editorconfig`_ in + your editor to prevent some linting warnings. + +6. Please avoid `unittest.assertTrue` or `unittest.assertFalse` when + possible, and use `assertEqual` or more specific assert. They give + better messages when it's failing, and are generally more correct. + .. _Google Python Style Guide: https://google.github.io/styleguide/pyguide.html .. _Sphinx-style: https://www.sphinx-doc.org/ .. _PEP 8 - Style Guide for Python Code: https://www.python.org/dev/peps/pep-0008 +.. _editorconfig: https://editorconfig.org/ Use ``certbot.compat.os`` instead of ``os`` =========================================== - Python's standard library ``os`` module lacks full support for several Windows security features about file permissions (eg. DACLs). However several files handled by Certbot (eg. private keys) need strongly restricted access @@ -460,11 +470,8 @@ Mypy type annotations ===================== Certbot uses the `mypy`_ static type checker. Python 3 natively supports official type annotations, -which can then be tested for consistency using mypy. Python 2 doesn’t, but type annotations can -be `added in comments`_. Mypy does some type checks even without type annotations; we can find -bugs in Certbot even without a fully annotated codebase. - -Certbot supports both Python 2 and 3, so we’re using Python 2-style annotations. +which can then be tested for consistency using mypy. Mypy does some type checks even without type +annotations; we can find bugs in Certbot even without a fully annotated codebase. Zulip wrote a `great guide`_ to using mypy. It’s useful, but you don’t have to read the whole thing to start contributing to Certbot. @@ -506,11 +513,13 @@ Steps: 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite including coverage. The ``--skip-missing-interpreters`` argument ignores missing versions of Python needed for running the tests. Fix any errors. -5. Submit the PR. Once your PR is open, please do not force push to the branch +5. If any documentation should be added or updated as part of the changes you + have made, please include the documentation changes in your PR. +6. Submit the PR. Once your PR is open, please do not force push to the branch containing your pull request to squash or amend commits. We use `squash merges `_ on PRs and rewriting commits makes changes harder to track between reviews. -6. Did your tests pass on Azure Pipelines? If they didn't, fix any errors. +7. Did your tests pass on Azure Pipelines? If they didn't, fix any errors. .. _ask for help: diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index c1d8cc403..8ae1c82f2 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -28,7 +28,7 @@ your system. System Requirements =================== -Certbot currently requires Python 2.7 or 3.6+ running on a UNIX-like operating +Certbot currently requires Python 3.6+ running on a UNIX-like operating system. By default, it requires root access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to port 80 (if you use the ``standalone`` plugin) and to read and @@ -44,17 +44,6 @@ supports `_ modern OSes based on Debian, Ubuntu, Fedora, SUSE, Gentoo and Darwin. - -Additional integrity verification of certbot-auto script can be done by verifying its digital signature. -This requires a local installation of gpg2, which comes packaged in many Linux distributions under name gnupg or gnupg2. - - -Installing with ``certbot-auto`` requires 512MB of RAM in order to build some -of the dependencies. Installing from pre-built OS packages avoids this -requirement. You can also temporarily set a swap file. See "Problems with -Python virtual environment" below for details. - - Alternate installation methods ================================ @@ -78,74 +67,6 @@ choosing "snapd" in the "System" dropdown menu. (You should select "snapd" regardless of your operating system, as our instructions are the same across all systems.) -.. _certbot-auto: - -Certbot-Auto ------------- - -The ``certbot-auto`` wrapper script installs Certbot, obtaining some dependencies -from your web server OS and putting others in a python virtual environment. You can -download and run it as follows:: - - wget https://dl.eff.org/certbot-auto - sudo mv certbot-auto /usr/local/bin/certbot-auto - sudo chown root /usr/local/bin/certbot-auto - sudo chmod 0755 /usr/local/bin/certbot-auto - /usr/local/bin/certbot-auto --help - -To remove certbot-auto, just delete it and the files it places under /opt/eff.org, along with any cronjob or systemd timer you may have created. - -To check the integrity of the ``certbot-auto`` script, -you can use these steps:: - - - user@webserver:~$ wget -N https://dl.eff.org/certbot-auto.asc - user@webserver:~$ gpg2 --keyserver pool.sks-keyservers.net --recv-key A2CFB51FA275A7286234E7B24D17C995CD9775F2 - user@webserver:~$ gpg2 --trusted-key 4D17C995CD9775F2 --verify certbot-auto.asc /usr/local/bin/certbot-auto - - - -The output of the last command should look something like:: - - - gpg: Signature made Wed 02 May 2018 05:29:12 AM IST - gpg: using RSA key A2CFB51FA275A7286234E7B24D17C995CD9775F2 - gpg: key 4D17C995CD9775F2 marked as ultimately trusted - gpg: checking the trustdb - gpg: marginals needed: 3 completes needed: 1 trust model: pgp - gpg: depth: 0 valid: 2 signed: 2 trust: 0-, 0q, 0n, 0m, 0f, 2u - gpg: depth: 1 valid: 2 signed: 0 trust: 2-, 0q, 0n, 0m, 0f, 0u - gpg: next trustdb check due at 2027-11-22 - gpg: Good signature from "Let's Encrypt Client Team " [ultimate] - - - -The ``certbot-auto`` command updates to the latest client release automatically. -Since ``certbot-auto`` is a wrapper to ``certbot``, it accepts exactly -the same command line flags and arguments. For more information, see -`Certbot command-line options `_. - -For full command line help, you can type:: - - /usr/local/bin/certbot-auto --help all - -Problems with Python virtual environment ----------------------------------------- - -On a low memory system such as VPS with less than 512MB of RAM, the required dependencies of Certbot will fail to build. -This can be identified if the pip outputs contains something like ``internal compiler error: Killed (program cc1)``. -You can workaround this restriction by creating a temporary swapfile:: - - user@webserver:~$ sudo fallocate -l 1G /tmp/swapfile - user@webserver:~$ sudo chmod 600 /tmp/swapfile - user@webserver:~$ sudo mkswap /tmp/swapfile - user@webserver:~$ sudo swapon /tmp/swapfile - -Disable and remove the swapfile once the virtual environment is constructed:: - - user@webserver:~$ sudo swapoff /tmp/swapfile - user@webserver:~$ sudo rm /tmp/swapfile - .. _docker-user: Running with Docker @@ -161,7 +82,7 @@ Docker if you are sure you know what you are doing and have a good reason to do so. You should definitely read the :ref:`where-certs` section, in order to -know how to manage the certs +know how to manage the certificates manually. `Our ciphersuites page `__ provides some information about recommended ciphersuites. If none of these make much sense to you, you should definitely use the installation method @@ -207,6 +128,18 @@ of the ``/etc/letsencrypt`` directory, see :ref:`where-certs`. Operating System Packages ------------------------- +.. warning:: While the Certbot team tries to keep the Certbot packages offered + by various operating systems working in the most basic sense, due to + distribution policies and/or the limited resources of distribution + maintainers, Certbot OS packages often have problems that other distribution + mechanisms do not. The packages are often old resulting in a lack of bug + fixes and features and a worse TLS configuration than is generated by newer + versions of Certbot. They also may not configure certificate renewal for you + or have all of Certbot's plugins available. For reasons like these, we + recommend most users follow the instructions at + https://certbot.eff.org/instructions and OS packages are only documented + here as an alternative. + **Arch Linux** .. code-block:: shell @@ -258,23 +191,23 @@ Optionally to install the Certbot Apache plugin, you can use: .. code-block:: shell - sudo apt-get install python-certbot-apache + sudo apt-get install python3-certbot-apache **Fedora** .. code-block:: shell - sudo dnf install certbot python2-certbot-apache + sudo dnf install certbot python3-certbot-apache **FreeBSD** * Port: ``cd /usr/ports/security/py-certbot && make install clean`` - * Package: ``pkg install py27-certbot`` + * Package: ``pkg install py37-certbot`` **Gentoo** -The official Certbot client is available in Gentoo Portage. From the -official Certbot plugins, three of them are also available in Portage. +The official Certbot client is available in Gentoo Portage. From the +official Certbot plugins, three of them are also available in Portage. They need to be installed separately if you require their functionality. .. code-block:: shell @@ -284,13 +217,13 @@ They need to be installed separately if you require their functionality. emerge -av app-crypt/certbot-nginx emerge -av app-crypt/certbot-dns-nsone -.. Note:: The ``app-crypt/certbot-dns-nsone`` package has a different +.. Note:: The ``app-crypt/certbot-dns-nsone`` package has a different maintainer than the other packages and can lag behind in version. **NetBSD** * Build from source: ``cd /usr/pkgsrc/security/py-certbot && make install clean`` - * Install pre-compiled package: ``pkg_add py27-certbot`` + * Install pre-compiled package: ``pkg_add py37-certbot`` **OpenBSD** @@ -303,6 +236,40 @@ OS packaging is an ongoing effort. If you'd like to package Certbot for your distribution of choice please have a look at the :doc:`packaging`. +.. _certbot-auto: + +Certbot-Auto +------------ +.. toctree:: + :hidden: + + uninstall + + +We used to have a shell script named ``certbot-auto`` to help people install +Certbot on UNIX operating systems, however, this script is no longer supported. +If you want to uninstall ``certbot-auto``, you can follow our instructions +:doc:`here `. + +Problems with Python virtual environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When using ``certbot-auto`` on a low memory system such as VPS with less than +512MB of RAM, the required dependencies of Certbot may fail to build. This can +be identified if the pip outputs contains something like ``internal compiler +error: Killed (program cc1)``. You can workaround this restriction by creating +a temporary swapfile:: + + user@webserver:~$ sudo fallocate -l 1G /tmp/swapfile + user@webserver:~$ sudo chmod 600 /tmp/swapfile + user@webserver:~$ sudo mkswap /tmp/swapfile + user@webserver:~$ sudo swapon /tmp/swapfile + +Disable and remove the swapfile once the virtual environment is constructed:: + + user@webserver:~$ sudo swapoff /tmp/swapfile + user@webserver:~$ sudo rm /tmp/swapfile + Installing from source ---------------------- diff --git a/certbot/docs/man/certbot.rst b/certbot/docs/man/certbot.rst index d03f3eed4..2f25504b0 100644 --- a/certbot/docs/man/certbot.rst +++ b/certbot/docs/man/certbot.rst @@ -1 +1,3 @@ +:orphan: + .. literalinclude:: ../cli-help.txt diff --git a/certbot/docs/uninstall.rst b/certbot/docs/uninstall.rst new file mode 100644 index 000000000..65151242c --- /dev/null +++ b/certbot/docs/uninstall.rst @@ -0,0 +1,16 @@ +========================= +Uninstalling certbot-auto +========================= + +To uninstall ``certbot-auto``, you need to do three things: + +1. If you added a cron job or systemd timer to automatically run + ``certbot-auto`` to renew your certificates, you should delete it. If you + did this by following our instructions, you can delete the entry added to + ``/etc/crontab`` by running a command like ``sudo sed -i '/certbot-auto/d' + /etc/crontab``. +2. Delete the ``certbot-auto`` script. If you placed it in ``/usr/local/bin`` + like we recommended, you can delete it by running ``sudo rm + /usr/local/bin/certbot-auto``. +3. Delete the Certbot installation created by ``certbot-auto`` by running + ``sudo rm -rf /opt/eff.org``. diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 290ac817a..1d97caecc 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -179,10 +179,9 @@ If you'd like to obtain a wildcard certificate from Let's Encrypt or run Certbot's DNS plugins. These plugins are not included in a default Certbot installation and must be -installed separately. While the DNS plugins cannot currently be used with -``certbot-auto``, they are available in many OS package managers, as Docker -images, and as snaps. Visit https://certbot.eff.org to learn the best way to use -the DNS plugins on your system. +installed separately. They are available in many OS package managers, as Docker +images, and as snaps. Visit https://certbot.eff.org to learn the best way to +use the DNS plugins on your system. Once installed, you can find documentation on how to use each plugin at: @@ -314,11 +313,12 @@ the ``certificates`` subcommand: This returns information in the following format:: - Found the following certs: + Found the following certificates: Certificate Name: example.com Domains: example.com, www.example.com Expiry Date: 2017-02-19 19:53:00+00:00 (VALID: 30 days) Certificate Path: /etc/letsencrypt/live/example.com/fullchain.pem + Key Type: RSA Private Key Path: /etc/letsencrypt/live/example.com/privkey.pem ``Certificate Name`` shows the name of the certificate. Pass this name @@ -346,7 +346,6 @@ control Certbot's behavior when re-creating a certificate with the same name as an existing certificate. If you don't specify a requested behavior, Certbot may ask you what you intended. - ``--force-renewal`` tells Certbot to request a new certificate with the same domains as an existing certificate. Each domain must be explicitly specified via ``-d``. If successful, this certificate @@ -380,7 +379,6 @@ If you prefer, you can specify the domains individually like this: Consider using ``--cert-name`` instead of ``--expand``, as it gives more control over which certificate is modified and it lets you remove domains as well as adding them. - ``--allow-subset-of-names`` tells Certbot to continue with certificate generation if only some of the specified domain authorizations can be obtained. This may be useful if some domains specified in a certificate no longer point at this @@ -411,32 +409,102 @@ replace that set entirely:: certbot certonly --cert-name example.com -d example.org,www.example.org +Using ECDSA keys +---------------- + +As of version 1.10, Certbot supports two types of private key algorithms: +``rsa`` and ``ecdsa``. The type of key used by Certbot can be controlled +through the ``--key-type`` option. You can also use the ``--elliptic-curve`` +option to control the curve used in ECDSA certificates. + +.. warning:: If you obtain certificates using ECDSA keys, you should be careful + not to downgrade your Certbot installation since ECDSA keys are not + supported by older versions of Certbot. Downgrades like this are possible if + you switch from something like the snaps or certbot-auto to packages + provided by your operating system which often lag behind. + +Changing existing certificates from RSA to ECDSA +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Unless you are aware that you need to support very old HTTPS clients that are +not supported by most sites, you can safely just transition your site to use +ECDSA keys instead of RSA keys. To accomplish this if you have existing +certificates managed by Certbot, you may freely change the certificate to a new +private key. + +If you want to use ECDSA keys for all certificates in the future, you can +simply add the following line to Certbot's :ref:`configuration file ` + +.. code-block:: ini + + key-type = ecdsa + +After this option is set, newly obtained certificates will use ECDSA keys. This +includes certificates managed by Certbot that previously used RSA keys. + +If you want to change a single certificate to use ECDSA keys, you'll need to +issue a new Certbot command setting ``--key-type ecdsa`` on the command line +like + +.. code-block:: shell + + certbot renew --key-type ecdsa --cert-name example.com --force-renewal + +Obtaining ECDSA certificates in addition to RSA certificates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When Certbot configures the certificates it obtains with Apache or Nginx, all +HTTPS clients that we try to support can use certificates with ECDSA keys. If, +however, you are aware of having a specific need to support very old TLS +clients, you may want to obtain both ECDSA and RSA certificates for the same +domains. Certbot can only configure Apache or Nginx to use a single +certificate, however, you could manually configure your software to use the +different certificates depending on your needs. + +When obtaining both ECDSA and RSA certificates for the same domains with +Certbot, we recommend using the ``--cert-name`` option to give your +certificates names so that you can easily identify them. For instance, you may +want to append "ecdsa" to the name of your ECDSA certificate by using a command +like + +.. code-block:: shell + + certbot certonly --key-type ecdsa --cert-name example.com-ecdsa + Revoking certificates --------------------- -If your account key has been compromised or you otherwise need to revoke a certificate, -use the ``revoke`` command to do so. Note that the ``revoke`` command takes the certificate path -(ending in ``cert.pem``), not a certificate name or domain. Example:: +If you need to revoke a certificate, use the ``revoke`` subcommand to do so. - certbot revoke --cert-path /etc/letsencrypt/live/CERTNAME/cert.pem +A certificate may be revoked by providing its name (see ``certbot certificates``) or by providing +its path directly:: + + certbot revoke --cert-name example.com + + certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem + +If the certificate being revoked was obtained via the ``--staging``, ``--test-cert`` or a non-default ``--server`` flag, +that flag must be passed to the ``revoke`` subcommand. + +.. note:: After revocation, Certbot will (by default) ask whether you want to **delete** the certificate. + Unless deleted, Certbot will try to renew revoked certificates the next time ``certbot renew`` runs. You can also specify the reason for revoking your certificate by using the ``reason`` flag. Reasons include ``unspecified`` which is the default, as well as ``keycompromise``, ``affiliationchanged``, ``superseded``, and ``cessationofoperation``:: - certbot revoke --cert-path /etc/letsencrypt/live/CERTNAME/cert.pem --reason keycompromise + certbot revoke --cert-name example.com --reason keycompromise -Additionally, if a certificate -is a test certificate obtained via the ``--staging`` or ``--test-cert`` flag, that flag must be passed to the -``revoke`` subcommand. -Once a certificate is revoked (or for other certificate management tasks), all of a certificate's -relevant files can be removed from the system with the ``delete`` subcommand:: +Revoking by account key or certificate private key +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - certbot delete --cert-name example.com +By default, Certbot will try revoke the certificate using your ACME account key. If the certificate was created from +the same ACME account, the revocation will be successful. -.. note:: If you don't use ``delete`` to remove the certificate completely, it will be renewed automatically at the next renewal event. +If you instead have the corresponding private key file to the certificate you wish to revoke, use ``--key-path`` to perform the +revocation from any ACME account:: -.. note:: Revoking a certificate will have no effect on the rate limit imposed by the Let's Encrypt server. + certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem --key-path /etc/letsencrypt/live/example.com/privkey.pem .. _renewal: @@ -852,7 +920,7 @@ Changing the ACME Server ======================== By default, Certbot uses Let's Encrypt's production server at -https://acme-v02.api.letsencrypt.org/. You can tell Certbot to use a +https://acme-v02.api.letsencrypt.org/directory. You can tell Certbot to use a different CA by providing ``--server`` on the command line or in a :ref:`configuration file ` with the URL of the server's ACME directory. For example, if you would like to use Let's Encrypt's diff --git a/certbot/examples/cli.ini b/certbot/examples/cli.ini index 4215fda5b..8471558ee 100644 --- a/certbot/examples/cli.ini +++ b/certbot/examples/cli.ini @@ -7,6 +7,10 @@ # certificate on a system with several certificates should not be placed # here. +# Use ECC for the private key +key-type = ecdsa +elliptic-curve = secp384r1 + # Use a 4096 bit RSA key instead of 2048 rsa-key-size = 4096 @@ -20,3 +24,11 @@ rsa-key-size = 4096 # path to the public_html / webroot folder being served by your web server. # authenticator = webroot # webroot-path = /usr/share/nginx/html + +# Uncomment to automatically agree to the terms of service of the ACME server +# agree-tos = true + +# An example of using an alternate ACME server that uses EAB credentials +# server = https://acme.sectigo.com/v2/InCommonRSAOV +# eab-kid = somestringofstuffwithoutquotes +# eab-hmac-key = yaddayaddahexhexnotquoted diff --git a/certbot/setup.py b/certbot/setup.py index 742900402..8b2874f20 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -31,25 +31,25 @@ meta = dict(re.findall(r"""__([a-z]+)__ = '([^']+)""", read_file(init_fn))) readme = read_file(os.path.join(here, 'README.rst')) version = meta['version'] -# This package relies on PyOpenSSL, requests, and six, however, it isn't -# specified here to avoid masking the more specific request requirements in -# acme. See https://github.com/pypa/pip/issues/988 for more info. +# This package relies on PyOpenSSL and requests, however, it isn't specified +# here to avoid masking the more specific request requirements in acme. See +# https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme>=1.8.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. 'ConfigArgParse>=0.9.3', - 'configobj', - 'cryptography>=1.2.3', # load_pem_x509_certificate + 'configobj>=5.0.6', + 'cryptography>=2.1.4', 'distro>=1.0.1', # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'parsedatetime>=1.3', # Calendar.parseDT + 'parsedatetime>=2.4', 'pyrfc3339', 'pytz', - 'setuptools', + 'setuptools>=39.0.1', 'zope.component', 'zope.interface', ] @@ -59,7 +59,7 @@ install_requires = [ # However environment markers are supported only with setuptools >= 36.2. # So this dependency is not added for old Linux distributions with old setuptools, # in order to allow these systems to build certbot from sources. -pywin32_req = 'pywin32>=227' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py +pywin32_req = 'pywin32>=300' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) if setuptools_known_environment_markers: install_requires.append(pywin32_req + " ; sys_platform == 'win32'") @@ -72,16 +72,14 @@ elif os.name == 'nt': # setuptools, pywin32 will not be specified as a dependency. install_requires.append(pywin32_req) -if setuptools_known_environment_markers: - install_requires.append('mock ; python_version < "3.3"') -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif sys.version_info < (3,3): - install_requires.append('mock') - dev_extras = [ + 'astroid', + 'azure-devops', 'coverage', + 'ipdb', + 'mypy', + 'PyGithub', + 'pylint', 'pytest', 'pytest-cov', 'pytest-xdist', @@ -90,15 +88,6 @@ dev_extras = [ 'wheel', ] -dev3_extras = [ - 'astroid', - 'azure-devops', - 'ipdb', - 'mypy', - 'PyGithub', - 'pylint', -] - docs_extras = [ # If you have Sphinx<1.5.1, you need docutils<0.13.1 # https://github.com/sphinx-doc/sphinx/issues/3212 @@ -116,7 +105,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=3.6', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', @@ -125,12 +114,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', @@ -145,7 +133,6 @@ setup( install_requires=install_requires, extras_require={ 'dev': dev_extras, - 'dev3': dev3_extras, 'docs': docs_extras, }, diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 7158827dc..d0448a1db 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -113,11 +113,16 @@ class AccountFileStorageTest(test_util.ConfigTestCase): from certbot._internal.account import Account new_authzr_uri = "hi" + meta = Account.Meta( + creation_host="test.example.org", + creation_dt=datetime.datetime( + 2021, 1, 5, 14, 4, 10, tzinfo=pytz.UTC)) self.acc = Account( regr=messages.RegistrationResource( uri=None, body=messages.Registration(), new_authzr_uri=new_authzr_uri), - key=KEY) + key=KEY, + meta=meta) self.mock_client = mock.MagicMock() self.mock_client.directory.new_authz = new_authzr_uri @@ -271,14 +276,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() mock_open.side_effect = IOError - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertRaises( errors.AccountStorageError, self.storage.load, self.acc.id) def test_save_ioerrors(self): mock_open = mock.mock_open() mock_open.side_effect = IOError # TODO: [None, None, IOError] - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): self.assertRaises( errors.AccountStorageError, self.storage.save, self.acc, self.mock_client) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 6cd207b4b..8eb5d7702 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -510,7 +510,7 @@ class ReportFailedAuthzrsTest(unittest.TestCase): auth_handler._report_failed_authzrs([self.authzr1], 'key') call_list = mock_zope().add_message.call_args_list - self.assertTrue(len(call_list) == 1) + self.assertEqual(len(call_list), 1) self.assertTrue("Domain: example.com\nType: tls\nDetail: detail" in call_list[0][0][0]) @test_util.patch_get_utility() @@ -518,7 +518,7 @@ class ReportFailedAuthzrsTest(unittest.TestCase): from certbot._internal import auth_handler auth_handler._report_failed_authzrs([self.authzr1, self.authzr2], 'key') - self.assertTrue(mock_zope().add_message.call_count == 2) + self.assertEqual(mock_zope().add_message.call_count, 2) def gen_auth_resp(chall_list): diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index a551e4400..ba6cfddc3 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -99,7 +99,7 @@ class UpdateLiveSymlinksTest(BaseCertManagerTest): for kind in ALL_FOUR: os.chdir(os.path.dirname(self.config_files[domain][kind])) self.assertEqual( - filesystem.realpath(os.readlink(self.config_files[domain][kind])), + filesystem.realpath(filesystem.readlink(self.config_files[domain][kind])), filesystem.realpath(archive_paths[domain][kind])) finally: os.chdir(prev_dir) @@ -526,7 +526,7 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config): from certbot._internal.cert_manager import cert_path_to_lineage @@ -556,21 +556,21 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): mock_acceptable_matches.return_value = [lambda x: x.cert_path] test_cert_path = os.path.join(self.config.config_dir, 'live', 'example.org', 'cert.pem') - self.config.cert_path = (test_cert_path, '') + self.config.cert_path = test_cert_path self.assertEqual('example.org', self._call(self.config)) @mock.patch('certbot._internal.cert_manager._acceptable_matches') def test_options_archive_cert(self, mock_acceptable_matches): # Also this and the next test check that the regex of _archive_files is working. - self.config.cert_path = (os.path.join(self.config.config_dir, 'archive', 'example.org', - 'cert11.pem'), '') + self.config.cert_path = os.path.join(self.config.config_dir, 'archive', 'example.org', + 'cert11.pem') mock_acceptable_matches.return_value = [lambda x: self._archive_files(x, 'cert')] self.assertEqual('example.org', self._call(self.config)) @mock.patch('certbot._internal.cert_manager._acceptable_matches') def test_options_archive_fullchain(self, mock_acceptable_matches): - self.config.cert_path = (os.path.join(self.config.config_dir, 'archive', - 'example.org', 'fullchain11.pem'), '') + self.config.cert_path = os.path.join(self.config.config_dir, 'archive', + 'example.org', 'fullchain11.pem') mock_acceptable_matches.return_value = [lambda x: self._archive_files(x, 'fullchain')] self.assertEqual('example.org', self._call(self.config)) @@ -586,7 +586,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config, acceptable_matches, match_func, rv_func): from certbot._internal.cert_manager import match_and_check_overlaps @@ -595,7 +595,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): def test_basic_match(self): from certbot._internal.cert_manager import _acceptable_matches self.assertEqual(['example.org'], self._call(self.config, _acceptable_matches(), - lambda x: self.config.cert_path[0], lambda x: x.lineagename)) + lambda x: self.config.cert_path, lambda x: x.lineagename)) @mock.patch('certbot._internal.cert_manager._search_lineages') def test_no_matches(self, mock_search_lineages): diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 7475e99ea..fca2b3e3e 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -1,16 +1,11 @@ """Tests for certbot._internal.cli.""" import argparse import copy +from importlib import reload as reload_module +import io import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six -from six.moves import reload_module # pylint: disable=import-error - from acme import challenges from certbot import errors from certbot._internal import cli @@ -21,6 +16,12 @@ from certbot.compat import os import certbot.tests.util as test_util from certbot.tests.util import TempDirTestCase +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + PLUGINS = disco.PluginsRegistry.find_all() @@ -91,7 +92,7 @@ class ParseTest(unittest.TestCase): def _help_output(self, args): "Run a command, and return the output string for scrutiny" - output = six.StringIO() + output = io.StringIO() def write_msg(message, *args, **kwargs): # pylint: disable=missing-docstring,unused-argument output.write(message) @@ -359,6 +360,21 @@ class ParseTest(unittest.TestCase): self.assertFalse(cli.option_was_set( 'authenticator', cli.flag_default('authenticator'))) + def test_ecdsa_key_option(self): + elliptic_curve_option = 'elliptic_curve' + elliptic_curve_option_value = cli.flag_default(elliptic_curve_option) + self.parse('--elliptic-curve {0}'.format(elliptic_curve_option_value).split()) + self.assertIs(cli.option_was_set(elliptic_curve_option, elliptic_curve_option_value), True) + + def test_invalid_key_type(self): + key_type_option = 'key_type' + key_type_value = cli.flag_default(key_type_option) + self.parse('--key-type {0}'.format(key_type_value).split()) + self.assertIs(cli.option_was_set(key_type_option, key_type_value), True) + + with self.assertRaises(SystemExit): + self.parse("--key-type foo") + def test_encode_revocation_reason(self): for reason, code in constants.REVOCATION_REASONS.items(): namespace = self.parse(['--reason', reason]) @@ -464,10 +480,6 @@ class ParseTest(unittest.TestCase): for topic in ['all', 'plugins', 'dns-route53']: self.assertFalse('certbot-route53:auth' in self._help_output([help_flag, topic])) - def test_no_permissions_check_accepted(self): - namespace = self.parse(["--no-permissions-check"]) - self.assertTrue(namespace.no_permissions_check) - class DefaultTest(unittest.TestCase): """Tests for certbot._internal.cli._Default.""" diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 1b2a7413e..f058cb658 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -13,7 +13,6 @@ except ImportError: # pragma: no cover from certbot import errors from certbot import util from certbot._internal import account -from certbot.compat import filesystem from certbot.compat import os import certbot.tests.util as test_util @@ -329,7 +328,11 @@ class ClientTest(ClientTestCommon): self._test_obtain_certificate_common(mock.sentinel.key, csr) mock_crypto_util.init_save_key.assert_called_once_with( - self.config.rsa_key_size, self.config.key_dir) + key_size=self.config.rsa_key_size, + key_dir=self.config.key_dir, + key_type=self.config.key_type, + elliptic_curve=None, # elliptic curve is not set + ) mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with( @@ -365,7 +368,11 @@ class ClientTest(ClientTestCommon): self.client.config.dry_run = True self._test_obtain_certificate_common(key, csr) - mock_crypto.make_key.assert_called_once_with(self.config.rsa_key_size) + mock_crypto.make_key.assert_called_once_with( + bits=self.config.rsa_key_size, + elliptic_curve=None, # not making an elliptic private key + key_type=self.config.key_type, + ) mock_acme_crypto.make_csr.assert_called_once_with( mock.sentinel.key_pem, self.eg_domains, self.config.must_staple) mock_crypto.init_save_key.assert_not_called() diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index 262fd02b5..ea48c9d1c 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -1,7 +1,6 @@ """Tests for certbot.compat.filesystem""" import contextlib import errno -import stat import unittest try: @@ -598,6 +597,32 @@ class IsExecutableTest(test_util.TempDirTestCase): self.assertFalse(filesystem.is_executable("exe")) +class ReadlinkTest(unittest.TestCase): + @unittest.skipUnless(POSIX_MODE, reason='Tests specific to Linux') + @mock.patch("certbot.compat.filesystem.os.readlink") + def test_path_posix(self, mock_readlink): + mock_readlink.return_value = "/normal/path" + self.assertEqual(filesystem.readlink("dummy"), "/normal/path") + + @unittest.skipIf(POSIX_MODE, reason='Tests specific to Windows') + @mock.patch("certbot.compat.filesystem.os.readlink") + def test_normal_path_windows(self, mock_readlink): + # Python <3.8 + mock_readlink.return_value = "C:\\short\\path" + self.assertEqual(filesystem.readlink("dummy"), "C:\\short\\path") + + # Python >=3.8 (os.readlink always returns the extended form) + mock_readlink.return_value = "\\\\?\\C:\\short\\path" + self.assertEqual(filesystem.readlink("dummy"), "C:\\short\\path") + + @unittest.skipIf(POSIX_MODE, reason='Tests specific to Windows') + @mock.patch("certbot.compat.filesystem.os.readlink") + def test_extended_path_windows(self, mock_readlink): + # Following path is largely over the 260 characters permitted in the normal form. + mock_readlink.return_value = "\\\\?\\C:\\long" + 1000 * "\\path" + with self.assertRaises(ValueError): + filesystem.readlink("dummy") + @contextlib.contextmanager def _fix_windows_runtime(): if os.name != 'nt': diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index bbd484c91..3b9c973f7 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -4,7 +4,7 @@ import unittest try: import mock -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover from unittest import mock import OpenSSL import zope.component @@ -32,6 +32,7 @@ CERT_LEAF = test_util.load_vector('cert_leaf.pem') CERT_ISSUER = test_util.load_vector('cert_intermediate_1.pem') CERT_ALT_ISSUER = test_util.load_vector('cert_intermediate_2.pem') + class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" def setUp(self): @@ -174,11 +175,56 @@ class ImportCSRFileTest(unittest.TestCase): class MakeKeyTest(unittest.TestCase): """Tests for certbot.crypto_util.make_key.""" - def test_it(self): # pylint: disable=no-self-use + def test_rsa(self): # pylint: disable=no-self-use + # RSA Key Type Test from certbot.crypto_util import make_key # Do not test larger keys as it takes too long. - OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, make_key(1024)) + OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, make_key(1024)) + + def test_ec(self): # pylint: disable=no-self-use + # ECDSA Key Type Tests + from certbot.crypto_util import make_key + + for (name, bits) in [('secp256r1', 256), ('secp384r1', 384), ('secp521r1', 521)]: + pkey = OpenSSL.crypto.load_privatekey( + OpenSSL.crypto.FILETYPE_PEM, + make_key(elliptic_curve=name, key_type='ecdsa') + ) + self.assertEqual(pkey.bits(), bits) + + def test_bad_key_sizes(self): + from certbot.crypto_util import make_key + # Try a bad key size for RSA and ECDSA + with self.assertRaises(errors.Error) as e: + make_key(bits=512, key_type='rsa') + self.assertEqual( + "Unsupported RSA key length: 512", + str(e.exception), + "Unsupported RSA key length: 512" + ) + + def test_bad_elliptic_curve_name(self): + from certbot.crypto_util import make_key + with self.assertRaises(errors.Error) as e: + make_key(elliptic_curve="nothere", key_type='ecdsa') + self.assertEqual( + "Unsupported elliptic curve: nothere", + str(e.exception), + "Unsupported elliptic curve: nothere" + ) + + def test_bad_key_type(self): + from certbot.crypto_util import make_key + + # Try a bad --key-type + with self.assertRaises(errors.Error) as e: + OpenSSL.crypto.load_privatekey( + OpenSSL.crypto.FILETYPE_PEM, make_key(1024, key_type='unf')) + self.assertEqual( + "Invalid key_type specified: unf. Use [rsa|ecdsa]", + str(e.exception), + "Invalid key_type specified: unf. Use [rsa|ecdsa]", + ) class VerifyCertSetup(unittest.TestCase): @@ -427,6 +473,19 @@ class FindChainWithIssuerTest(unittest.TestCase): matched = self._call(fullchains, "Pebble Root CA 0cc6f0") self.assertEqual(matched, fullchains[1]) + @mock.patch('certbot.crypto_util.logger.info') + def test_intermediate_match(self, mock_info): + """Don't pick a chain where only an intermediate matches""" + fullchains = self._all_fullchains() + # Make the second chain actually only contain "Pebble Root CA 0cc6f0" + # as an intermediate, not as the root. This wouldn't be a valid chain + # (the CERT_ISSUER cert didn't issue the CERT_ALT_ISSUER cert), but the + # function under test here doesn't care about that. + fullchains[1] = fullchains[1] + CERT_ISSUER.decode() + matched = self._call(fullchains, "Pebble Root CA 0cc6f0") + self.assertEqual(matched, fullchains[0]) + mock_info.assert_not_called() + @mock.patch('certbot.crypto_util.logger.info') def test_no_match(self, mock_info): fullchains = self._all_fullchains() diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 0852ab175..8ae6290bf 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -3,19 +3,20 @@ try: import readline # pylint: disable=import-error except ImportError: import certbot._internal.display.dummy_readline as readline # type: ignore +from importlib import reload as reload_module import string import sys import unittest +from certbot.compat import filesystem # pylint: disable=ungrouped-imports +from certbot.compat import os # pylint: disable=ungrouped-imports +import certbot.tests.util as test_util # pylint: disable=ungrouped-imports + try: import mock except ImportError: # pragma: no cover from unittest import mock -from six.moves import reload_module # pylint: disable=import-error -from certbot.compat import filesystem # pylint: disable=ungrouped-imports -from certbot.compat import os # pylint: disable=ungrouped-imports -import certbot.tests.util as test_util # pylint: disable=ungrouped-imports class CompleterTest(test_util.TempDirTestCase): diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index cdbcf1937..ce60a5f0b 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -43,7 +43,7 @@ class GetEmailTest(unittest.TestCase): mock_input.return_value = (display_util.OK, "foo@bar.baz") with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.return_value = True - self.assertTrue(self._call() == "foo@bar.baz") + self.assertEqual(self._call(), "foo@bar.baz") @test_util.patch_get_utility("certbot.display.ops.z_util") def test_ok_not_safe(self, mock_get_utility): @@ -51,7 +51,7 @@ class GetEmailTest(unittest.TestCase): mock_input.return_value = (display_util.OK, "foo@bar.baz") with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.side_effect = [False, True] - self.assertTrue(self._call() == "foo@bar.baz") + self.assertEqual(self._call(), "foo@bar.baz") @test_util.patch_get_utility("certbot.display.ops.z_util") def test_invalid_flag(self, mock_get_utility): diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 7af454abd..30b33bbc9 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -1,20 +1,22 @@ """Test :mod:`certbot.display.util`.""" import inspect +import io import socket import tempfile import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six from certbot import errors from certbot import interfaces from certbot.display import util as display_util import certbot.tests.util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + CHOICES = [("First", "Description1"), ("Second", "Description2")] TAGS = ["tag1", "tag2", "tag3"] TAGS_CHOICES = [("1", "tag1"), ("2", "tag2"), ("3", "tag3")] @@ -34,7 +36,7 @@ class InputWithTimeoutTest(unittest.TestCase): def test_input(self, prompt=None): expected = "foo bar" - stdin = six.StringIO(expected + "\n") + stdin = io.StringIO(expected + "\n") with mock.patch("certbot.compat.misc.select.select") as mock_select: mock_select.return_value = ([stdin], [], [],) self.assertEqual(self._call(prompt), expected) @@ -328,11 +330,7 @@ class FileOutputDisplayTest(unittest.TestCase): # Every IDisplay method implemented by FileDisplay must take # force_interactive to prevent workflow regressions. for name in interfaces.IDisplay.names(): - if six.PY2: - getargspec = inspect.getargspec - else: - getargspec = inspect.getfullargspec - arg_spec = getargspec(getattr(self.displayer, name)) # pylint: disable=deprecated-method + arg_spec = inspect.getfullargspec(getattr(self.displayer, name)) self.assertTrue("force_interactive" in arg_spec.args) @@ -402,12 +400,8 @@ class NoninteractiveDisplayTest(unittest.TestCase): for name in interfaces.IDisplay.names(): # pylint: disable=E1120 method = getattr(self.displayer, name) # asserts method accepts arbitrary keyword arguments - if six.PY2: - result = inspect.getargspec(method).keywords # pylint:deprecated-method - self.assertFalse(result is None) - else: - result = inspect.getfullargspec(method).varkw - self.assertFalse(result is None) + result = inspect.getfullargspec(method).varkw + self.assertFalse(result is None) class SeparateListInputTest(unittest.TestCase): @@ -454,6 +448,27 @@ class PlaceParensTest(unittest.TestCase): self.assertEqual("(y)es please", self._call("yes please")) +class SummarizeDomainListTest(unittest.TestCase): + @classmethod + def _call(cls, domains): + from certbot.display.util import summarize_domain_list + return summarize_domain_list(domains) + + def test_single_domain(self): + self.assertEqual("example.com", self._call(["example.com"])) + + def test_two_domains(self): + self.assertEqual("example.com and example.org", + self._call(["example.com", "example.org"])) + + def test_many_domains(self): + self.assertEqual("example.com and 2 more domains", + self._call(["example.com", "example.org", "a.example.com"])) + + def test_empty_domains(self): + self.assertEqual("", self._call([])) + + class NotifyTest(unittest.TestCase): """Test the notify function """ @@ -462,7 +477,7 @@ class NotifyTest(unittest.TestCase): from certbot.display.util import notify notify("Hello World") mock_util().notification.assert_called_with( - "Hello World", pause=False, decorate=False + "Hello World", pause=False, decorate=False, wrap=False ) diff --git a/certbot/tests/helpful_test.py b/certbot/tests/helpful_test.py index 292e55304..1a5c2bea6 100644 --- a/certbot/tests/helpful_test.py +++ b/certbot/tests/helpful_test.py @@ -1,6 +1,11 @@ """Tests for certbot.helpful_parser""" import unittest +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + from certbot import errors from certbot._internal.cli import HelpfulArgumentParser from certbot._internal.cli import _DomainsAction @@ -189,5 +194,16 @@ class TestParseArgsErrors(unittest.TestCase): arg_parser.parse_args() +class TestAddDeprecatedArgument(unittest.TestCase): + """Tests for add_deprecated_argument method of HelpfulArgumentParser""" + + @mock.patch.object(HelpfulArgumentParser, "modify_kwargs_for_default_detection") + def test_no_default_detection_modifications(self, mock_modify): + arg_parser = HelpfulArgumentParser(["run"], {}, detect_defaults=True) + arg_parser.add_deprecated_argument("--foo", 0) + arg_parser.parse_args() + mock_modify.assert_not_called() + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 76764e61b..088faa0fb 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -1,15 +1,11 @@ """Tests for certbot._internal.log.""" +import io import logging import logging.handlers import sys import time import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six from acme import messages from certbot import errors @@ -19,6 +15,12 @@ from certbot.compat import filesystem from certbot.compat import os from certbot.tests import util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + class PreArgParseSetupTest(unittest.TestCase): """Tests for certbot._internal.log.pre_arg_parse_setup.""" @@ -75,7 +77,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): self.devnull = open(os.devnull, 'w') from certbot._internal.log import ColoredStreamHandler - self.stream_handler = ColoredStreamHandler(six.StringIO()) + self.stream_handler = ColoredStreamHandler(io.StringIO()) from certbot._internal.log import MemoryHandler, TempHandler self.temp_handler = TempHandler() self.temp_path = self.temp_handler.path @@ -179,7 +181,7 @@ class ColoredStreamHandlerTest(unittest.TestCase): """Tests for certbot._internal.log.ColoredStreamHandler""" def setUp(self): - self.stream = six.StringIO() + self.stream = io.StringIO() self.stream.isatty = lambda: True self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) @@ -213,7 +215,7 @@ class MemoryHandlerTest(unittest.TestCase): self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.DEBUG) self.msg = 'hi there' - self.stream = six.StringIO() + self.stream = io.StringIO() self.stream_handler = logging.StreamHandler(self.stream) from certbot._internal.log import MemoryHandler @@ -238,7 +240,7 @@ class MemoryHandlerTest(unittest.TestCase): def test_target_reset(self): self._test_log_debug() - new_stream = six.StringIO() + new_stream = io.StringIO() new_stream_handler = logging.StreamHandler(new_stream) self.handler.setTarget(new_stream_handler) self.handler.flush(force=True) @@ -325,7 +327,7 @@ class PostArgParseExceptHookTest(unittest.TestCase): def test_acme_error(self): # Get an arbitrary error code - acme_code = next(six.iterkeys(messages.ERROR_CODES)) + acme_code = next(iter(messages.ERROR_CODES)) def get_acme_error(msg): """Wraps ACME errors so the constructor takes only a msg.""" @@ -349,7 +351,7 @@ class PostArgParseExceptHookTest(unittest.TestCase): def _test_common(self, error_type, debug): """Returns the mocked logger and stderr output.""" - mock_err = six.StringIO() + mock_err = io.StringIO() def write_err(*args, **unused_kwargs): """Write error to mock_err.""" diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 63307009d..7668a1b6a 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,9 +1,9 @@ # coding=utf-8 """Tests for certbot._internal.main.""" # pylint: disable=too-many-lines -from __future__ import print_function - import datetime +from importlib import reload as reload_module +import io import itertools import json import shutil @@ -13,13 +13,7 @@ import traceback import unittest import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock import pytz -import six -from six.moves import reload_module # pylint: disable=import-error from certbot import crypto_util from certbot import errors @@ -39,6 +33,12 @@ from certbot.compat import os from certbot.plugins import enhancements import certbot.tests.util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + CERT_PATH = test_util.vector_path('cert_512.pem') CERT = test_util.vector_path('cert_512.pem') @@ -49,14 +49,51 @@ RSA2048_KEY_PATH = test_util.vector_path('rsa2048_key.pem') SS_CERT_PATH = test_util.vector_path('cert_2048.pem') -class TestHandleIdenticalCerts(unittest.TestCase): - """Test for certbot._internal.main._handle_identical_cert_request""" - def test_handle_identical_cert_request_pending(self): +class TestHandleCerts(unittest.TestCase): + """Test for certbot._internal.main._handle_* methods""" + @mock.patch("certbot._internal.main._handle_unexpected_key_type_migration") + def test_handle_identical_cert_request_pending(self, mock_handle_migration): mock_lineage = mock.Mock() mock_lineage.ensure_deployed.return_value = False # pylint: disable=protected-access ret = main._handle_identical_cert_request(mock.Mock(), mock_lineage) self.assertEqual(ret, ("reinstall", mock_lineage)) + self.assertTrue(mock_handle_migration.called) + + @mock.patch("certbot._internal.main._handle_unexpected_key_type_migration") + def test_handle_subset_cert_request(self, mock_handle_migration): + mock_config = mock.Mock() + mock_config.expand = True + mock_lineage = mock.Mock() + mock_lineage.names.return_value = ["dummy1", "dummy2"] + ret = main._handle_subset_cert_request(mock_config, ["dummy1"], mock_lineage) + self.assertEqual(ret, ("renew", mock_lineage)) + self.assertTrue(mock_handle_migration.called) + + @mock.patch("certbot._internal.main.cli.set_by_cli") + def test_handle_unexpected_key_type_migration(self, mock_set): + config = mock.Mock() + config.key_type = "rsa" + cert = mock.Mock() + cert.private_key_type = "ecdsa" + + mock_set.return_value = True + main._handle_unexpected_key_type_migration(config, cert) + + mock_set.return_value = False + with self.assertRaises(errors.Error) as raised: + main._handle_unexpected_key_type_migration(config, cert) + self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + + mock_set.side_effect = lambda var: var != "certname" + with self.assertRaises(errors.Error) as raised: + main._handle_unexpected_key_type_migration(config, cert) + self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + + mock_set.side_effect = lambda var: var != "key_type" + with self.assertRaises(errors.Error) as raised: + main._handle_unexpected_key_type_migration(config, cert) + self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) class RunTest(test_util.ConfigTestCase): @@ -163,31 +200,35 @@ class CertonlyTest(unittest.TestCase): @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @mock.patch('certbot._internal.cert_manager.domains_for_certname') @mock.patch('certbot._internal.renewal.renew_cert') + @mock.patch('certbot._internal.main._handle_unexpected_key_type_migration') @mock.patch('certbot._internal.main._report_new_cert') def test_find_lineage_for_domains_and_certname(self, mock_report_cert, - mock_renew_cert, mock_domains, mock_lineage): + mock_handle_type, mock_renew_cert, mock_domains, mock_lineage): domains = ['example.com', 'test.org'] mock_domains.return_value = domains mock_lineage.names.return_value = domains self._call(('certonly --webroot -d example.com -d test.org ' '--cert-name example.com').split()) - self.assertTrue(mock_lineage.call_count == 1) - self.assertTrue(mock_domains.call_count == 1) - self.assertTrue(mock_renew_cert.call_count == 1) - self.assertTrue(mock_report_cert.call_count == 1) + + self.assertEqual(mock_lineage.call_count, 1) + self.assertEqual(mock_domains.call_count, 1) + self.assertEqual(mock_renew_cert.call_count, 1) + self.assertEqual(mock_report_cert.call_count, 1) + self.assertEqual(mock_handle_type.call_count, 1) # user confirms updating lineage with new domains self._call(('certonly --webroot -d example.com -d test.com ' '--cert-name example.com').split()) - self.assertTrue(mock_lineage.call_count == 2) - self.assertTrue(mock_domains.call_count == 2) - self.assertTrue(mock_renew_cert.call_count == 2) - self.assertTrue(mock_report_cert.call_count == 2) + self.assertEqual(mock_lineage.call_count, 2) + self.assertEqual(mock_domains.call_count, 2) + self.assertEqual(mock_renew_cert.call_count, 2) + self.assertEqual(mock_report_cert.call_count, 2) + self.assertEqual(mock_handle_type.call_count, 2) # error in _ask_user_to_confirm_new_names self.mock_get_utility().yesno.return_value = False self.assertRaises(errors.ConfigurationError, self._call, - ('certonly --webroot -d example.com -d test.com --cert-name example.com').split()) + 'certonly --webroot -d example.com -d test.com --cert-name example.com'.split()) @mock.patch('certbot._internal.cert_manager.domains_for_certname') @mock.patch('certbot.display.ops.choose_names') @@ -200,14 +241,15 @@ class CertonlyTest(unittest.TestCase): # no lineage with this name but we specified domains so create a new cert self._call(('certonly --webroot -d example.com -d test.com ' '--cert-name example.com').split()) - self.assertTrue(mock_lineage.call_count == 1) - self.assertTrue(mock_report_cert.call_count == 1) + self.assertEqual(mock_lineage.call_count, 1) + self.assertEqual(mock_report_cert.call_count, 1) # no lineage with this name and we didn't give domains mock_choose_names.return_value = ["somename"] mock_domains_for_certname.return_value = None self._call(('certonly --webroot --cert-name example.com').split()) - self.assertTrue(mock_choose_names.called) + self.assertIs(mock_choose_names.called, True) + class FindDomainsOrCertnameTest(unittest.TestCase): """Tests for certbot._internal.main._find_domains_or_certname.""" @@ -243,10 +285,7 @@ class RevokeTest(test_util.TempDirTestCase): super(RevokeTest, self).setUp() shutil.copy(CERT_PATH, self.tempdir) - self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, - 'cert_512.pem')) - with open(self.tmp_cert_path, 'r') as f: - self.tmp_cert = (self.tmp_cert_path, f.read()) + self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, 'cert_512.pem')) patches = [ mock.patch('acme.client.BackwardsCompatibleClientV2'), @@ -276,6 +315,7 @@ class RevokeTest(test_util.TempDirTestCase): if not args: args = 'revoke --cert-path={0} ' args = args.format(self.tmp_cert_path).split() + cli.set_by_cli.detector = None # required to reset set_by_cli state plugins = disco.PluginsRegistry.find_all() config = configuration.NamespaceConfig( cli.prepare_and_parse_args(plugins, args)) @@ -301,13 +341,44 @@ class RevokeTest(test_util.TempDirTestCase): self.assertEqual(expected, mock_revoke.call_args_list) @mock.patch('certbot._internal.main._delete_if_appropriate') - @mock.patch('certbot._internal.storage.cert_path_for_cert_name') - def test_revoke_by_certname(self, mock_cert_path_for_cert_name, - mock_delete_if_appropriate): + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, + server="https://acme.example") args = 'revoke --cert-name=example.com'.split() - mock_cert_path_for_cert_name.return_value = self.tmp_cert mock_delete_if_appropriate.return_value = False self._call(args) + self.mock_acme_client.assert_called_once_with(mock.ANY, mock.ANY, 'https://acme.example') + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + + @mock.patch('certbot._internal.main._delete_if_appropriate') + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname_and_server(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + """Revoking with --server should use the server from the CLI""" + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, + server="https://acme.example") + args = 'revoke --cert-name=example.com --server https://other.example'.split() + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_acme_client.assert_called_once_with(mock.ANY, mock.ANY, 'https://other.example') + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + + @mock.patch('certbot._internal.main._delete_if_appropriate') + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname_empty_server(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + """Revoking with --cert-name where the lineage server is empty shouldn't crash """ + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, server=None) + args = 'revoke --cert-name=example.com'.split() + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_acme_client.assert_called_once_with( + mock.ANY, mock.ANY, constants.CLI_DEFAULTS['server']) self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) @mock.patch('certbot._internal.main._delete_if_appropriate') @@ -556,7 +627,7 @@ class MainTest(test_util.ConfigTestCase): "Run the client with output streams mocked out" args = self.standard_args + args - toy_stdout = stdout if stdout else six.StringIO() + toy_stdout = stdout if stdout else io.StringIO() with mock.patch('certbot._internal.main.sys.stdout', new=toy_stdout): with mock.patch('certbot._internal.main.sys.stderr') as stderr: with mock.patch("certbot.util.atexit"): @@ -569,8 +640,8 @@ class MainTest(test_util.ConfigTestCase): self.assertEqual(1, mock_run.call_count) def test_version_string_program_name(self): - toy_out = six.StringIO() - toy_err = six.StringIO() + toy_out = io.StringIO() + toy_err = io.StringIO() with mock.patch('certbot._internal.main.sys.stdout', new=toy_out): with mock.patch('certbot._internal.main.sys.stderr', new=toy_err): try: @@ -718,7 +789,7 @@ class MainTest(test_util.ConfigTestCase): # This needed two calls to find_all(), which we're avoiding for now # because of possible side effects: # https://github.com/letsencrypt/letsencrypt/commit/51ed2b681f87b1eb29088dd48718a54f401e4855 - #with mock.patch('certbot._internal.cli.plugins_testable') as plugins: + # with mock.patch('certbot._internal.cli.plugins_testable') as plugins: # plugins.return_value = {"apache": True, "nginx": True} # ret, _, _, _ = self._call(args) # self.assertTrue("Too many flags setting" in ret) @@ -771,12 +842,14 @@ class MainTest(test_util.ConfigTestCase): self._call_no_clientmock(['delete']) self.assertEqual(1, mock_cert_manager.call_count) + @mock.patch('certbot._internal.main.plugins_disco') + @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') @mock.patch('certbot._internal.log.post_arg_parse_setup') - def test_plugins(self, _): + def test_plugins(self, _, _det, mock_disco): flags = ['--init', '--prepare', '--authenticators', '--installers'] for args in itertools.chain( *(itertools.combinations(flags, r) - for r in six.moves.range(len(flags)))): + for r in range(len(flags)))): self._call(['plugins'] + list(args)) @mock.patch('certbot._internal.main.plugins_disco') @@ -785,7 +858,7 @@ class MainTest(test_util.ConfigTestCase): ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() - stdout = six.StringIO() + stdout = io.StringIO() with test_util.patch_get_utility_with_stdout(stdout=stdout): _, stdout, _, _ = self._call(['plugins'], stdout) @@ -805,7 +878,7 @@ class MainTest(test_util.ConfigTestCase): _, _, _ = directory, mode, strict raise errors.Error() - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('certbot.util.set_up_core_dir') as mock_set_up_core_dir: with test_util.patch_get_utility_with_stdout(stdout=stdout): mock_set_up_core_dir.side_effect = throw_error @@ -822,7 +895,7 @@ class MainTest(test_util.ConfigTestCase): ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() - stdout = six.StringIO() + stdout = io.StringIO() with test_util.patch_get_utility_with_stdout(stdout=stdout): _, stdout, _, _ = self._call(['plugins', '--init'], stdout) @@ -840,7 +913,7 @@ class MainTest(test_util.ConfigTestCase): ifaces = [] # type: List[interfaces.IPlugin] plugins = mock_disco.PluginsRegistry.find_all() - stdout = six.StringIO() + stdout = io.StringIO() with test_util.patch_get_utility_with_stdout(stdout=stdout): _, stdout, _, _ = self._call(['plugins', '--init', '--prepare'], stdout) @@ -985,10 +1058,11 @@ class MainTest(test_util.ConfigTestCase): mock_lineage.should_autorenew.return_value = due_for_renewal mock_lineage.has_pending_deployment.return_value = False mock_lineage.names.return_value = ['isnot.org'] + mock_lineage.private_key_type = 'RSA' mock_certr = mock.MagicMock() mock_key = mock.MagicMock(pem='pem_key') mock_client = mock.MagicMock() - stdout = six.StringIO() + stdout = io.StringIO() mock_client.obtain_certificate.return_value = (mock_certr, 'chain', mock_key, 'csr') @@ -1006,7 +1080,7 @@ class MainTest(test_util.ConfigTestCase): mock_get_utility().notification.side_effect = write_msg with mock.patch('certbot._internal.main.renewal.OpenSSL') as mock_ssl: mock_latest = mock.MagicMock() - mock_latest.get_issuer.return_value = "Fake fake" + mock_latest.get_issuer.return_value = "Artificial pretend" mock_ssl.crypto.load_certificate.return_value = mock_latest with mock.patch('certbot._internal.main.renewal.crypto_util') \ as mock_crypto_util: @@ -1126,7 +1200,7 @@ class MainTest(test_util.ConfigTestCase): _, _, stdout = self._test_renewal_common(False, extra_args=None, should_renew=False, args=['renew'], expiry_date=expiry) self.assertTrue('No renewals were attempted.' in stdout.getvalue()) - self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue()) + self.assertTrue('The following certificates are not due for renewal yet:' in stdout.getvalue()) @mock.patch('certbot._internal.log.post_arg_parse_setup') def test_quiet_renew(self, _): @@ -1277,7 +1351,7 @@ class MainTest(test_util.ConfigTestCase): _, _, stdout = self._test_renewal_common( due_for_renewal=False, extra_args=None, should_renew=False, args=['renew', '--post-hook', - '{0} -c "from __future__ import print_function; print(\'hello world\');"' + '{0} -c "print(\'hello world\');"' .format(sys.executable)]) self.assertTrue('No hooks were run.' in stdout.getvalue()) @@ -1401,85 +1475,6 @@ class MainTest(test_util.ConfigTestCase): x = self._call_no_clientmock(["register", "--email", "user@example.org"]) self.assertTrue("There is an existing account" in x[0]) - def test_update_account_no_existing_accounts(self): - # with mock.patch('certbot._internal.main.client') as mocked_client: - with mock.patch('certbot._internal.main.account') as mocked_account: - mocked_storage = mock.MagicMock() - mocked_account.AccountFileStorage.return_value = mocked_storage - mocked_storage.find_all.return_value = [] - x = self._call_no_clientmock( - ["update_account", "--email", - "user@example.org"]) - self.assertTrue("Could not find an existing account" in x[0]) - - @mock.patch('certbot._internal.main._determine_account') - @mock.patch('certbot._internal.eff.prepare_subscription') - @mock.patch('certbot._internal.main.account') - def test_update_account_remove_email(self, mocked_account_module, mock_prepare, mock_det_acc): - # Mock account storage and the account object returned - mocked_storage = mock.MagicMock() - mocked_account = mock.MagicMock() - - mocked_account_module.AccountFileStorage.return_value = mocked_storage - mocked_storage.find_all.return_value = [mocked_account] - mock_det_acc.return_value = (mocked_account, "foo") - - # Mock registration body to verify calls are made - mock_regr_body = mock.MagicMock() - - # mocked_account.regr is overwritten in update, requiring an odd mock setup - mocked_account.regr.body = mock_regr_body - - x = self._call( - ["update_account", "--register-unsafely-without-email"]) - - - # When update succeeds, the return value of update_account() is None - self.assertTrue(x[0] is None) - # and we got supposedly did update the registration from - # the server - client_mock = x[3] - self.assertTrue(client_mock.Client().acme.update_registration.called) - - self.assertTrue(mock_regr_body.update.called) - self.assertTrue('contact' in mock_regr_body.update.call_args[1]) - self.assertEqual(mock_regr_body.update.call_args[1]['contact'], ()) - # and we saved the updated registration on disk - self.assertTrue(mocked_storage.update_regr.called) - # ensure we didn't try to subscribe (no email to subscribe with) - self.assertFalse(mock_prepare.called) - - @mock.patch("certbot._internal.main.display_util.notify") - @mock.patch('certbot._internal.main.display_ops.get_email') - @test_util.patch_get_utility() - def test_update_account_with_email(self, mock_utility, mock_email, mock_notify): - email = "user@example.com" - mock_email.return_value = email - with mock.patch('certbot._internal.eff.prepare_subscription') as mock_prepare: - with mock.patch('certbot._internal.main._determine_account') as mocked_det: - with mock.patch('certbot._internal.main.account') as mocked_account: - with mock.patch('certbot._internal.main.client') as mocked_client: - mocked_storage = mock.MagicMock() - mocked_account.AccountFileStorage.return_value = mocked_storage - mocked_storage.find_all.return_value = ["an account"] - mocked_det.return_value = (mock.MagicMock(), "foo") - cb_client = mock.MagicMock() - mocked_client.Client.return_value = cb_client - x = self._call_no_clientmock( - ["update_account"]) - # When registration change succeeds, the return value - # of register() is None - self.assertTrue(x[0] is None) - # and we got supposedly did update the registration from - # the server - self.assertTrue( - cb_client.acme.update_registration.called) - # and we saved the updated registration on disk - self.assertTrue(mocked_storage.update_regr.called) - self.assertTrue( - email in mock_notify.call_args[0][0]) - self.assertTrue(mock_prepare.called) - @mock.patch('certbot._internal.plugins.selection.choose_configurator_plugins') @mock.patch('certbot._internal.updater._run_updaters') def test_plugin_selection_error(self, mock_run, mock_choose): @@ -1752,5 +1747,111 @@ class InstallTest(test_util.ConfigTestCase): self.config, plugins) +class UpdateAccountTest(test_util.ConfigTestCase): + """Tests for certbot._internal.main.update_account""" + + def setUp(self): + patches = { + 'account': mock.patch('certbot._internal.main.account'), + 'atexit': mock.patch('certbot.util.atexit'), + 'client': mock.patch('certbot._internal.main.client'), + 'determine_account': mock.patch('certbot._internal.main._determine_account'), + 'notify': mock.patch('certbot._internal.main.display_util.notify'), + 'prepare_sub': mock.patch('certbot._internal.eff.prepare_subscription'), + 'util': test_util.patch_get_utility() + } + self.mocks = { k: patches[k].start() for k in patches } + for patch in patches.values(): + self.addCleanup(patch.stop) + + return super(UpdateAccountTest, self).setUp() + + def _call(self, args): + with mock.patch('certbot._internal.main.sys.stdout'), \ + mock.patch('certbot._internal.main.sys.stderr'): + args = ['--config-dir', self.config.config_dir, + '--work-dir', self.config.work_dir, + '--logs-dir', self.config.logs_dir, '--text'] + args + return main.main(args[:]) # NOTE: parser can alter its args! + + def _prepare_mock_account(self): + mock_storage = mock.MagicMock() + mock_account = mock.MagicMock() + mock_regr = mock.MagicMock() + mock_storage.find_all.return_value = [mock_account] + self.mocks['account'].AccountFileStorage.return_value = mock_storage + mock_account.regr.body = mock_regr.body + self.mocks['determine_account'].return_value = (mock_account, mock.MagicMock()) + return (mock_account, mock_storage, mock_regr) + + def _test_update_no_contact(self, args): + """Utility to assert that email removal is handled correctly""" + (_, mock_storage, mock_regr) = self._prepare_mock_account() + result = self._call(args) + # When update succeeds, the return value of update_account() is None + self.assertIsNone(result) + # We submitted a registration to the server + self.assertEqual(self.mocks['client'].Client().acme.update_registration.call_count, 1) + mock_regr.body.update.assert_called_with(contact=()) + # We got an update from the server and persisted it + self.assertEqual(mock_storage.update_regr.call_count, 1) + # We should have notified the user + self.mocks['notify'].assert_called_with( + 'Any contact information associated with this account has been removed.' + ) + # We should not have called subscription because there's no email + self.mocks['prepare_sub'].assert_not_called() + + def test_no_existing_accounts(self): + """Test that no existing account is handled correctly""" + mock_storage = mock.MagicMock() + mock_storage.find_all.return_value = [] + self.mocks['account'].AccountFileStorage.return_value = mock_storage + self.assertEqual(self._call(['update_account', '--email', 'user@example.org']), + 'Could not find an existing account to update.') + + def test_update_account_remove_email(self): + """Test that --register-unsafely-without-email is handled as no email""" + self._test_update_no_contact(['update_account', '--register-unsafely-without-email']) + + def test_update_account_empty_email(self): + """Test that providing an empty email is handled as no email""" + self._test_update_no_contact(['update_account', '-m', '']) + + @mock.patch('certbot._internal.main.display_ops.get_email') + def test_update_account_with_email(self, mock_email): + """Test that updating with a singular email is handled correctly""" + mock_email.return_value = 'user@example.com' + (_, mock_storage, _) = self._prepare_mock_account() + mock_client = mock.MagicMock() + self.mocks['client'].Client.return_value = mock_client + + result = self._call(['update_account']) + # None if registration succeeds + self.assertIsNone(result) + # We should have updated the server + self.assertEqual(mock_client.acme.update_registration.call_count, 1) + # We should have updated the account on disk + self.assertEqual(mock_storage.update_regr.call_count, 1) + # Subscription should have been prompted + self.assertEqual(self.mocks['prepare_sub'].call_count, 1) + # Should have printed the email + self.mocks['notify'].assert_called_with( + 'Your e-mail address was updated to user@example.com.') + + def test_update_account_with_multiple_emails(self): + """Test that multiple email addresses are handled correctly""" + (_, mock_storage, mock_regr) = self._prepare_mock_account() + self.assertIsNone( + self._call(['update_account', '-m', 'user@example.com,user@example.org']) + ) + mock_regr.body.update.assert_called_with( + contact=['mailto:user@example.com', 'mailto:user@example.org'] + ) + self.assertEqual(mock_storage.update_regr.call_count, 1) + self.mocks['notify'].assert_called_with( + 'Your e-mail address was updated to user@example.com,user@example.org.') + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 344e6312f..bcd190e9b 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -233,11 +233,11 @@ class AddrTest(unittest.TestCase): def test_eq(self): self.assertEqual(self.addr1, self.addr2.get_addr_obj("")) self.assertNotEqual(self.addr1, self.addr2) - self.assertFalse(self.addr1 == 3333) + self.assertNotEqual(self.addr1, 3333) self.assertEqual(self.addr4, self.addr4.get_addr_obj("")) self.assertNotEqual(self.addr4, self.addr5) - self.assertFalse(self.addr4 == 3333) + self.assertNotEqual(self.addr4, 3333) from certbot.plugins.common import Addr self.assertEqual(self.addr4, Addr.fromstring("[fe00:0:0::1]")) self.assertEqual(self.addr4, Addr.fromstring("[fe00:0::0:0:1]")) diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index ed13544de..a97de24b9 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -8,7 +8,6 @@ try: except ImportError: # pragma: no cover from unittest import mock import pkg_resources -import six import zope.interface from certbot import errors @@ -56,7 +55,7 @@ class PluginEntryPointTest(unittest.TestCase): EP_SA: "sa", } - for entry_point, name in six.iteritems(names): + for entry_point, name in names.items(): self.assertEqual( name, PluginEntryPoint.entry_point_to_plugin_name(entry_point, with_prefix=False)) @@ -70,7 +69,7 @@ class PluginEntryPointTest(unittest.TestCase): self.ep3: "p3:ep3", } - for entry_point, name in six.iteritems(names): + for entry_point, name in names.items(): self.assertEqual( name, PluginEntryPoint.entry_point_to_plugin_name(entry_point, with_prefix=True)) diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index 993f3b461..31761e986 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -29,7 +29,7 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen def more_info(self): # pylint: disable=missing-docstring,no-self-use return 'A fake authenticator for testing.' - class _FakeConfig(object): + class _FakeConfig: fake_propagation_seconds = 0 fake_config_key = 1 fake_other_key = None diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 7318783fd..0e552e0c5 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -6,7 +6,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six from acme import challenges from certbot import errors @@ -53,7 +52,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.assertRaises(errors.HookCommandNotFound, self.auth.prepare) def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) + self.assertTrue(isinstance(self.auth.more_info(), str)) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref('example.org'), @@ -61,7 +60,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): def test_script_perform(self): self.config.manual_auth_hook = ( - '{0} -c "from __future__ import print_function;' + '{0} -c "' 'from certbot.compat import os;' 'print(os.environ.get(\'CERTBOT_DOMAIN\'));' 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index 47708e340..dad5b270a 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -5,7 +5,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six class InstallerTest(unittest.TestCase): @@ -16,7 +15,7 @@ class InstallerTest(unittest.TestCase): self.installer = Installer(config=mock.MagicMock(), name="null") def test_it(self): - self.assertTrue(isinstance(self.installer.more_info(), six.string_types)) + self.assertTrue(isinstance(self.installer.more_info(), str)) self.assertEqual([], self.installer.get_all_names()) self.assertEqual([], self.installer.supported_enhancements()) diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 751b9d943..596cee622 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -10,7 +10,6 @@ try: except ImportError: # pragma: no cover from unittest import mock import OpenSSL.crypto # pylint: disable=unused-import -import six from acme import challenges from acme import standalone as acme_standalone # pylint: disable=unused-import @@ -91,7 +90,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.servers = mock.MagicMock() def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) + self.assertTrue(isinstance(self.auth.more_info(), str)) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref(domain=None), diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index 4b0d1da83..2999d306e 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -33,7 +33,7 @@ class PluginStorageTest(test_util.ConfigTestCase): mock_open.side_effect = IOError self.plugin.storage.storagepath = os.path.join(self.config.config_dir, ".pluginstorage.json") - with mock.patch("six.moves.builtins.open", mock_open): + with mock.patch("builtins.open", mock_open): with mock.patch('certbot.compat.os.path.isfile', return_value=True): with mock.patch("certbot.reverter.util"): self.assertRaises(errors.PluginStorageError, diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index e57e09eae..e6fbd8e88 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -14,7 +14,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six from acme import challenges from certbot import achallenges @@ -59,7 +58,7 @@ class AuthenticatorTest(unittest.TestCase): def test_more_info(self): more_info = self.auth.more_info() - self.assertTrue(isinstance(more_info, six.string_types)) + self.assertTrue(isinstance(more_info, str)) self.assertTrue(self.path in more_info) def test_add_parser_arguments(self): @@ -83,7 +82,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertTrue(self.achall.domain in call[0][0]) self.assertTrue(all( webroot in call[0][1] - for webroot in six.itervalues(self.config.webroot_map))) + for webroot in self.config.webroot_map.values())) self.assertEqual(self.config.webroot_map[self.achall.domain], self.path) @@ -100,7 +99,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertTrue(self.achall.domain in call[0][0]) self.assertTrue(all( webroot in call[0][1] - for webroot in six.itervalues(self.config.webroot_map))) + for webroot in self.config.webroot_map.values())) @test_util.patch_get_utility() def test_new_webroot(self, mock_get_utility): diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index 0957b5c31..7c9c53fb4 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -1,4 +1,6 @@ """Tests for certbot._internal.renewal""" +import copy + import unittest try: @@ -74,6 +76,47 @@ class RenewalTest(test_util.ConfigTestCase): assert self.config.rsa_key_size == 2048 + def test_reuse_ec_key_renewal_params(self): + self.config.elliptic_curve = 'INVALID_CURVE' + self.config.reuse_key = True + self.config.dry_run = True + self.config.key_type = 'ecdsa' + config = configuration.NamespaceConfig(self.config) + + rc_path = test_util.make_lineage( + self.config.config_dir, + 'sample-renewal-ec.conf', + ec=True, + ) + lineage = storage.RenewableCert(rc_path, config) + + le_client = mock.MagicMock() + le_client.obtain_certificate.return_value = (None, None, None, None) + + from certbot._internal import renewal + + with mock.patch('certbot._internal.renewal.hooks.renew_hook'): + renewal.renew_cert(self.config, None, le_client, lineage) + + assert self.config.elliptic_curve == 'secp256r1' + + @test_util.patch_get_utility() + @mock.patch('certbot._internal.renewal.cli.set_by_cli') + def test_remove_deprecated_config_elements(self, mock_set_by_cli, unused_mock_get_utility): + mock_set_by_cli.return_value = False + config = configuration.NamespaceConfig(self.config) + config.certname = "sample-renewal-deprecated-option" + + rc_path = test_util.make_lineage( + self.config.config_dir, 'sample-renewal-deprecated-option.conf') + + from certbot._internal import renewal + lineage_config = copy.deepcopy(self.config) + renewal_candidate = renewal._reconstitute(lineage_config, rc_path) + # This means that manual_public_ip_logging_ok was not modified in the config based on its + # value in the renewal conf file + self.assertTrue(isinstance(lineage_config.manual_public_ip_logging_ok, mock.MagicMock)) + class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): """Tests for certbot._internal.renewal.restore_required_config_elements.""" @@ -139,5 +182,70 @@ class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): self.assertEqual(self.config.server, constants.CLI_DEFAULTS['server']) +class DescribeResultsTest(unittest.TestCase): + """Tests for certbot._internal.renewal._renew_describe_results.""" + def setUp(self): + self.patchers = { + 'log_error': mock.patch('certbot._internal.renewal.logger.error'), + 'notify': mock.patch('certbot._internal.renewal.display_util.notify')} + self.mock_notify = self.patchers['notify'].start() + self.mock_error = self.patchers['log_error'].start() + + def tearDown(self): + for patch in self.patchers.values(): + patch.stop() + + @classmethod + def _call(cls, *args, **kwargs): + from certbot._internal.renewal import _renew_describe_results + _renew_describe_results(*args, **kwargs) + + def _assert_success_output(self, lines): + self.mock_notify.assert_has_calls([mock.call(l) for l in lines]) + + def test_no_renewal_attempts(self): + self._call(mock.MagicMock(dry_run=True), [], [], [], []) + self._assert_success_output(['No simulated renewals were attempted.']) + + def test_successful_renewal(self): + self._call(mock.MagicMock(dry_run=False), ['good.pem'], None, None, None) + self._assert_success_output([ + '\n- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + 'Congratulations, all renewals succeeded: ', + ' good.pem (success)', + '- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + ]) + + def test_failed_renewal(self): + self._call(mock.MagicMock(dry_run=False), [], ['bad.pem'], [], []) + self._assert_success_output([ + '\n- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + '- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + ]) + self.mock_error.assert_has_calls([ + mock.call('All %ss failed. The following certificates could not be renewed:', 'renewal'), + mock.call(' bad.pem (failure)'), + ]) + + def test_all_renewal(self): + self._call(mock.MagicMock(dry_run=True), + ['good.pem', 'good2.pem'], ['bad.pem', 'bad2.pem'], + ['foo.pem expires on 123'], ['errored.conf']) + self._assert_success_output([ + '\n- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + 'The following certificates are not due for renewal yet:', + ' foo.pem expires on 123 (skipped)', + 'The following simulated renewals succeeded:', + ' good.pem (success)\n good2.pem (success)\n', + '\nAdditionally, the following renewal configurations were invalid: ', + ' errored.conf (parsefail)', + '- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + ]) + self.mock_error.assert_has_calls([ + mock.call('The following %ss failed:', 'simulated renewal'), + mock.call(' bad.pem (failure)\n bad2.pem (failure)'), + ]) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 7d03f1821..7a37f782e 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -1,12 +1,13 @@ """Tests for certbot._internal.reporter.""" +import io import sys import unittest + try: import mock except ImportError: # pragma: no cover from unittest import mock -import six class ReporterTest(unittest.TestCase): @@ -16,7 +17,7 @@ class ReporterTest(unittest.TestCase): self.reporter = reporter.Reporter(mock.MagicMock(quiet=False)) self.old_stdout = sys.stdout # type: ignore - sys.stdout = six.StringIO() + sys.stdout = io.StringIO() def tearDown(self): sys.stdout = self.old_stdout diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index d67aa431a..af01a9a1b 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -9,7 +9,6 @@ try: import mock except ImportError: # pragma: no cover from unittest import mock -import six from certbot import errors from certbot.compat import os @@ -156,7 +155,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): act_coms = get_undo_commands(self.config.temp_checkpoint_dir) - for a_com, com in six.moves.zip(act_coms, coms): + for a_com, com in zip(act_coms, coms): self.assertEqual(a_com, com) def test_bad_register_undo_command(self): diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index b67c4cbce..1e3a1ffd8 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -11,7 +11,6 @@ try: except ImportError: # pragma: no cover from unittest import mock import pytz -import six import certbot from certbot import errors @@ -77,6 +76,17 @@ class RelevantValuesTest(unittest.TestCase): self.assertEqual(self._call(self.values), expected_relevant_values) + @mock.patch("certbot._internal.cli.set_by_cli") + def test_deprecated_item(self, unused_mock_set_by_cli): + # deprecated items should never be relevant to store + expected_relevant_values = self.values.copy() + self.values["manual_public_ip_logging_ok"] = None + self.assertEqual(self._call(self.values), expected_relevant_values) + self.values["manual_public_ip_logging_ok"] = True + self.assertEqual(self._call(self.values), expected_relevant_values) + self.values["manual_public_ip_logging_ok"] = False + self.assertEqual(self._call(self.values), expected_relevant_values) + class BaseRenewableCertTest(test_util.ConfigTestCase): """Base class for setting up Renewable Cert tests. @@ -275,7 +285,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version("cert"), None) def test_latest_and_next_versions(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(self.test_rc.latest_common_version(), 5) @@ -315,7 +325,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_update_link_to(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) @@ -330,7 +340,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc._update_link_to("chain", 3000) # However, current_version doesn't allow querying the resulting # version (because it's a broken link). - self.assertEqual(os.path.basename(os.readlink(self.test_rc.chain)), + self.assertEqual(os.path.basename(filesystem.readlink(self.test_rc.chain)), "chain3000.pem") def test_version(self): @@ -342,12 +352,12 @@ class RenewableCertTests(BaseRenewableCertTest): os.path.basename(self.test_rc.version("cert", 8))) def test_update_all_links_to_success(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) self.assertEqual(self.test_rc.latest_common_version(), 5) - for ver in six.moves.range(1, 6): + for ver in range(1, 6): self.test_rc.update_all_links_to(ver) for kind in ALL_FOUR: self.assertEqual(ver, self.test_rc.current_version(kind)) @@ -385,11 +395,11 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version(kind), 11) def test_has_pending_deployment(self): - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) - for ver in six.moves.range(1, 6): + for ver in range(1, 6): self.test_rc.update_all_links_to(ver) for kind in ALL_FOUR: self.assertEqual(ver, self.test_rc.current_version(kind)) @@ -487,7 +497,7 @@ class RenewableCertTests(BaseRenewableCertTest): # (to avoid instantiating parser) mock_rv.side_effect = lambda x: x - for ver in six.moves.range(1, 6): + for ver in range(1, 6): for kind in ALL_FOUR: self._write_out_kind(kind, ver) self.test_rc.update_all_links_to(3) @@ -514,11 +524,11 @@ class RenewableCertTests(BaseRenewableCertTest): # privkey. for i in (6, 7, 8): self.assertTrue(os.path.islink(self.test_rc.version("privkey", i))) - self.assertEqual("privkey3.pem", os.path.basename(os.readlink( + self.assertEqual("privkey3.pem", os.path.basename(filesystem.readlink( self.test_rc.version("privkey", i)))) for kind in ALL_FOUR: - self.assertEqual(self.test_rc.available_versions(kind), list(six.moves.range(1, 9))) + self.assertEqual(self.test_rc.available_versions(kind), list(range(1, 9))) self.assertEqual(self.test_rc.current_version(kind), 3) # Test updating from latest version rather than old version self.test_rc.update_all_links_to(8) @@ -527,7 +537,7 @@ class RenewableCertTests(BaseRenewableCertTest): b'attempt', self.config)) for kind in ALL_FOUR: self.assertEqual(self.test_rc.available_versions(kind), - list(six.moves.range(1, 10))) + list(range(1, 10))) self.assertEqual(self.test_rc.current_version(kind), 8) with open(self.test_rc.version("fullchain", 9)) as f: self.assertEqual(f.read(), "last" + "attempt") @@ -755,6 +765,13 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(storage.add_time_interval(base_time, interval), excepted) + def test_server(self): + self.test_rc.configuration["renewalparams"] = {} + self.assertEqual(self.test_rc.server, None) + rp = self.test_rc.configuration["renewalparams"] + rp["server"] = "https://acme.example/dir" + self.assertEqual(self.test_rc.server, "https://acme.example/dir") + def test_is_test_cert(self): self.test_rc.configuration["renewalparams"] = {} rp = self.test_rc.configuration["renewalparams"] @@ -924,14 +941,14 @@ class CertPathForCertNameTest(BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config, certname): from certbot._internal.storage import cert_path_for_cert_name return cert_path_for_cert_name(cli_config, certname) def test_simple_cert_name(self): - self.assertEqual(self._call(self.config, 'example.org'), (self.fullchain, 'fullchain')) + self.assertEqual(self._call(self.config, 'example.org'), self.fullchain) def test_no_such_cert_name(self): self.assertRaises(errors.CertStorageError, self._call, self.config, 'fake-example.org') diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index d28d7df6f..18947c342 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -1,21 +1,22 @@ """Tests for certbot.util.""" import argparse import errno +from importlib import reload as reload_module +import io import sys import unittest -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock -import six -from six.moves import reload_module # pylint: disable=import-error - from certbot import errors from certbot.compat import filesystem from certbot.compat import os import certbot.tests.util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + + class EnvNoSnapForExternalCallsTest(unittest.TestCase): """Tests for certbot.util.env_no_snap_for_external_calls.""" @@ -127,7 +128,7 @@ class LockDirUntilExit(test_util.TempDirTestCase): from certbot import util # Despite lock_dir_until_exit has been called twice to subdir, its lock should have been # added only once. So we expect to have two lock references: for self.tempdir and subdir - self.assertTrue(len(util._LOCKS) == 2) # pylint: disable=protected-access + self.assertEqual(len(util._LOCKS), 2) # pylint: disable=protected-access registered_func() # Exception should not be raised # Logically, logger.debug, that would be invoked in case of unlock failure, # should never been called. @@ -265,11 +266,11 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): def test_multiple(self): items = [] - for _ in six.moves.range(10): + for _ in range(10): items.append(self._call("wow")) f, name = items[-1] self.assertTrue(isinstance(f, file_type)) - self.assertTrue(isinstance(name, six.string_types)) + self.assertTrue(isinstance(name, str)) self.assertTrue("wow-0009.conf" in name) for f, _ in items: f.close() @@ -361,7 +362,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): def test_help(self): self._call("--old-option", 2) - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch("sys.stdout", new=stdout): try: self.parser.parse_args(["-h"]) @@ -547,8 +548,7 @@ class OsInfoTest(unittest.TestCase): m_distro.linux_distribution.return_value = ("something", "else") self.assertEqual(cbutil.get_os_info(), ("something", "else")) - @mock.patch("certbot.util.subprocess.Popen") - def test_non_systemd_os_info(self, popen_mock): + def test_non_systemd_os_info(self): import certbot.util as cbutil with mock.patch('certbot.util._USE_DISTRO', False): with mock.patch('platform.system_alias', @@ -557,13 +557,14 @@ class OsInfoTest(unittest.TestCase): with mock.patch('platform.system_alias', return_value=('darwin', '', '')): - comm_mock = mock.Mock() - comm_attrs = {'communicate.return_value': - ('42.42.42', 'error')} - comm_mock.configure_mock(**comm_attrs) - popen_mock.return_value = comm_mock - self.assertEqual(cbutil.get_python_os_info()[0], 'darwin') - self.assertEqual(cbutil.get_python_os_info()[1], '42.42.42') + with mock.patch("subprocess.Popen") as popen_mock: + comm_mock = mock.Mock() + comm_attrs = {'communicate.return_value': + ('42.42.42', 'error')} + comm_mock.configure_mock(**comm_attrs) + popen_mock.return_value = comm_mock + self.assertEqual(cbutil.get_python_os_info()[0], 'darwin') + self.assertEqual(cbutil.get_python_os_info()[1], '42.42.42') with mock.patch('platform.system_alias', return_value=('freebsd', '9.3-RC3-p1', '')): diff --git a/letsencrypt-auto b/letsencrypt-auto index 2a0cda9b3..24af0b4c3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.9.0" +LE_AUTO_VERSION="1.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -799,15 +799,14 @@ BootstrapMageiaCommon() { # that function. If Bootstrap is set to a function that doesn't install any # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then - Bootstrap() { - BootstrapMessage "Debian-based OSes" - BootstrapDebCommon - } - BOOTSTRAP_VERSION="BootstrapDebCommon $BOOTSTRAP_DEB_COMMON_VERSION" + DEPRECATED_OS=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then + DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -840,12 +839,7 @@ elif [ -f /etc/redhat-release ]; then INTERACTIVE_BOOTSTRAP=1 fi - Bootstrap() { - BootstrapMessage "Legacy RedHat-based OSes that will use Python3" - BootstrapRpmPython3Legacy - } USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION" # Try now to enable SCL rh-python36 for systems already bootstrapped # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto @@ -864,43 +858,38 @@ elif [ -f /etc/redhat-release ]; then fi if [ "$RPM_USE_PYTHON_3" = 1 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/arch-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/manjaro-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/gentoo-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif uname | grep -iq FreeBSD ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif uname | grep -iq Darwin ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then - Bootstrap() { - ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 else DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 fi # We handle this case after determining the normal bootstrap version to allow @@ -1129,7 +1118,9 @@ if [ "$1" = "--le-auto-phase2" ]; then fi if [ -f "$VENV_BIN/letsencrypt" -a "$INSTALL_ONLY" != 1 ]; then - error "Certbot will no longer receive updates." + error "certbot-auto and its Certbot installation will no longer receive updates." + error "You will not receive any bug fixes including those fixing server compatibility" + error "or security problems." error "Please visit https://certbot.eff.org/ to check for other alternatives." "$VENV_BIN/letsencrypt" "$@" exit 0 @@ -1497,18 +1488,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.9.0 \ - --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \ - --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57 -acme==1.9.0 \ - --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \ - --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f -certbot-apache==1.9.0 \ - --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \ - --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e -certbot-nginx==1.9.0 \ - --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \ - --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65 +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/Dockerfile.redhat6 b/letsencrypt-auto-source/Dockerfile.redhat6 deleted file mode 100644 index 66f21bc14..000000000 --- a/letsencrypt-auto-source/Dockerfile.redhat6 +++ /dev/null @@ -1,54 +0,0 @@ -# For running tests, build a docker image with a passwordless sudo and a trust -# store we can manipulate. - -ARG REDHAT_DIST_FLAVOR -FROM ${REDHAT_DIST_FLAVOR}:6 - -ARG REDHAT_DIST_FLAVOR - -RUN curl -O https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm \ - && rpm -ivh epel-release-latest-6.noarch.rpm - -# Install pip and sudo: -RUN yum install -y python-pip sudo -# Update to a stable and tested version of pip. -# We do not use pipstrap here because it no longer supports Python 2.6. -RUN pip install pip==9.0.1 setuptools==29.0.1 wheel==0.29.0 -# Pin pytest version for increased stability -RUN pip install pytest==3.2.5 six==1.10.0 - -# Add an unprivileged user: -RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups wheel --uid 1000 lea - -# Let that user sudo: -RUN sed -i.bkp -e \ - 's/# %wheel\(NOPASSWD: ALL\)\?/%wheel/g' \ - /etc/sudoers - -RUN mkdir -p /home/lea/certbot - -# Install fake testing CA: -COPY ./tests/certs/ca/my-root-ca.crt.pem /usr/local/share/ca-certificates/ -RUN update-ca-trust - -# Copy current letsencrypt-auto: -COPY . /home/lea/certbot/letsencrypt-auto-source - -# Tweak uname binary for tests on fake 32bits -COPY tests/uname_wrapper.sh /bin -RUN mv /bin/uname /bin/uname_orig \ - && mv /bin/uname_wrapper.sh /bin/uname \ - && chmod +x /bin/uname - -# Fetch previous letsencrypt-auto that was installing python 3.4 -RUN curl https://raw.githubusercontent.com/certbot/certbot/v0.38.0/letsencrypt-auto-source/letsencrypt-auto \ - -o /home/lea/certbot/letsencrypt-auto-source/letsencrypt-auto_py_34 \ - && chmod +x /home/lea/certbot/letsencrypt-auto-source/letsencrypt-auto_py_34 - -RUN cp /home/lea/certbot/letsencrypt-auto-source/tests/${REDHAT_DIST_FLAVOR}6_tests.sh /home/lea/certbot/letsencrypt-auto-source/tests/redhat6_tests.sh \ - && chmod +x /home/lea/certbot/letsencrypt-auto-source/tests/redhat6_tests.sh - -USER lea -WORKDIR /home/lea - -CMD ["sudo", "certbot/letsencrypt-auto-source/tests/redhat6_tests.sh"] diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index f73e8b35e..fbd16f2b0 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl98wk8ACgkQTRfJlc2X -dfIctgf/TO83xXJJ8haqxke0ehHCwcmipX7ijPhwvaUTSqciMa56KnGJLNp1lAVz -vv8sfHUf7NSvGlRg+5M0szWY25+JzveJDNzse3rOzFmxA1GNKUycE3/zE/IdBRwN -fmxJHaUBrBL2erBZPHe8gFGTvlzopBoGSmQpWGY3hIufPWKBJohCbTscKbaa9hyz -njmMvwRdeqzvLWVZ4jNDDsil9kKl2Emue3guzA/cvVxHe17DZyLDfqni7ysZIcTn -wPAQzpLBKHyiqVRoVk+BJ6Z6wamW4NAxKbjXy9GrHy4txlfW8tGd3jXha8yWqJeH -xEFK02Zp+T17+C5uqEW4o0cIofMjCw== -=9UGf +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmA+sv4ACgkQTRfJlc2X +dfLG+ggAvwUCqy06UZd8jZhOZFUoi8nWHG2TMnm/4CW4G9PPCjsCoQplhaAaUCrR +PAv0vtsG1rBKWekICg6IBTzLVioH9xRPUkpfbVQhT1c1T/5CqMsFFXR5p9YAKRe7 +hlOb7VRN10bdCS4JThRPNhdWFdWKZXYKcIOObWA/FX2GacxMHuLwPpSsbt2NRffy +qz1ZOWxvr289aWEbZWfyBiI2bxQ7wlMEbZ/JLUXDe46ETDxzENu+c0x2109ha6m4 +wHmUS0r16ps/n9DueTZGJf3C26mU+cIB+LgNvOcibBo+0Ly7t+OiBHYbkFXS2KTQ +RbH4bPZrsduUOzhE8wIsSUIsVGDleQ== +=jYnL -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 83a1041ba..b0b25759f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.10.0.dev0" +LE_AUTO_VERSION="1.14.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -800,10 +800,14 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then + DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -836,12 +840,7 @@ elif [ -f /etc/redhat-release ]; then INTERACTIVE_BOOTSTRAP=1 fi - Bootstrap() { - BootstrapMessage "Legacy RedHat-based OSes that will use Python3" - BootstrapRpmPython3Legacy - } USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION" # Try now to enable SCL rh-python36 for systems already bootstrapped # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto @@ -860,43 +859,38 @@ elif [ -f /etc/redhat-release ]; then fi if [ "$RPM_USE_PYTHON_3" = 1 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi fi LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/arch-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/manjaro-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/gentoo-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif uname | grep -iq FreeBSD ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif uname | grep -iq Darwin ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then - Bootstrap() { - ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 else DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 fi # We handle this case after determining the normal bootstrap version to allow @@ -1125,7 +1119,9 @@ if [ "$1" = "--le-auto-phase2" ]; then fi if [ -f "$VENV_BIN/letsencrypt" -a "$INSTALL_ONLY" != 1 ]; then - error "Certbot will no longer receive updates." + error "certbot-auto and its Certbot installation will no longer receive updates." + error "You will not receive any bug fixes including those fixing server compatibility" + error "or security problems." error "Please visit https://certbot.eff.org/ to check for other alternatives." "$VENV_BIN/letsencrypt" "$@" exit 0 @@ -1493,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.9.0 \ - --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \ - --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57 -acme==1.9.0 \ - --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \ - --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f -certbot-apache==1.9.0 \ - --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \ - --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e -certbot-nginx==1.9.0 \ - --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \ - --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65 +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 66b2b2084..3516f77b0 100644 --- a/letsencrypt-auto-source/letsencrypt-auto.sig +++ b/letsencrypt-auto-source/letsencrypt-auto.sig @@ -1,2 +1,3 @@ -zP]m*>Ouj0x}R ʆ&EA`*̉9KEcZs66\i L} -d9h%qG2*yfT)%ƓfsKn :.`gq&iz C5w"Хr6%H-״ F\h*p*^D%~z|/L 1b:#}ԝALK.Qn! ^[U^֪0Då}*fzRA'- \ No newline at end of file +_̊ПN* && docker run --rm -t -i lea - -""" diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py deleted file mode 100644 index 805bb21af..000000000 --- a/letsencrypt-auto-source/tests/auto_test.py +++ /dev/null @@ -1,503 +0,0 @@ -"""Tests for letsencrypt-auto""" - -from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler -from contextlib import contextmanager -from functools import partial -from json import dumps -from os import chmod, environ, makedirs, stat -from os.path import abspath, dirname, exists, join -import re -from shutil import copy, rmtree -import socket -import ssl -from stat import S_IMODE, S_IRUSR, S_IWUSR, S_IXUSR, S_IWGRP, S_IWOTH -from subprocess import CalledProcessError, Popen, PIPE -import sys -from tempfile import mkdtemp -from threading import Thread -from unittest import TestCase - -from pytest import mark -from six.moves import xrange # pylint: disable=redefined-builtin - - -@mark.skip -def tests_dir(): - """Return a path to the "tests" directory.""" - return dirname(abspath(__file__)) - - -def copy_stable(src, dst): - """ - Copy letsencrypt-auto, and replace its current version to its equivalent stable one. - This is needed to test correctly the self-upgrade functionality. - """ - copy(src, dst) - with open(dst, 'r') as file: - filedata = file.read() - filedata = re.sub(r'LE_AUTO_VERSION="(.*)\.dev0"', r'LE_AUTO_VERSION="\1"', filedata) - with open(dst, 'w') as file: - file.write(filedata) - - -sys.path.insert(0, dirname(tests_dir())) -from build import build as build_le_auto - - -BOOTSTRAP_FILENAME = 'certbot-auto-bootstrap-version.txt' -"""Name of the file where certbot-auto saves its bootstrap version.""" - - -class RequestHandler(BaseHTTPRequestHandler): - """An HTTPS request handler which is quiet and serves a specific folder.""" - - def __init__(self, resources, *args, **kwargs): - """ - :arg resources: A dict of resource paths pointing to content bytes - - """ - self.resources = resources - BaseHTTPRequestHandler.__init__(self, *args, **kwargs) - - def log_message(self, format, *args): - """Don't log each request to the terminal.""" - - def do_GET(self): - """Serve a GET request.""" - content = self.send_head() - if content is not None: - self.wfile.write(content) - - def send_head(self): - """Common code for GET and HEAD commands - - This sends the response code and MIME headers and returns either a - bytestring of content or, if none is found, None. - - """ - path = self.path[1:] # Strip leading slash. - content = self.resources.get(path) - if content is None: - self.send_error(404, 'Path "%s" not found in self.resources' % path) - else: - self.send_response(200) - self.send_header('Content-type', 'text/plain') - self.send_header('Content-Length', str(len(content))) - self.end_headers() - return content - - -def server_and_port(resources): - """Return an unstarted HTTPS server and the port it will use.""" - # Find a port, and bind to it. I can't get the OS to close the socket - # promptly after we shut down the server, so we typically need to try - # a couple ports after the first test case. Setting - # TCPServer.allow_reuse_address = True seems to have nothing to do - # with this behavior. - worked = False - for port in xrange(4443, 4543): - try: - server = HTTPServer(('localhost', port), - partial(RequestHandler, resources)) - except socket.error: - pass - else: - worked = True - server.socket = ssl.wrap_socket( - server.socket, - certfile=join(tests_dir(), 'certs', 'localhost', 'server.pem'), - server_side=True) - break - if not worked: - raise RuntimeError("Couldn't find an unused socket for the testing HTTPS server.") - return server, port - - -@contextmanager -def serving(resources): - """Spin up a local HTTPS server, and yield its base URL. - - Use a self-signed cert generated as outlined by - https://coolaj86.com/articles/create-your-own-certificate-authority-for- - testing/. - - """ - server, port = server_and_port(resources) - thread = Thread(target=server.serve_forever) - try: - thread.start() - yield 'https://localhost:{port}/'.format(port=port) - finally: - server.shutdown() - thread.join() - - -LE_AUTO_PATH = join(dirname(tests_dir()), 'letsencrypt-auto') - - -@contextmanager -def temp_paths(): - """Creates and deletes paths for letsencrypt-auto and its venv.""" - dir = mkdtemp(prefix='le-test-') - try: - yield join(dir, 'letsencrypt-auto'), join(dir, 'venv') - finally: - rmtree(dir, ignore_errors=True) - - -def out_and_err(command, input=None, shell=False, env=None): - """Run a shell command, and return stderr and stdout as string. - - If the command returns nonzero, raise CalledProcessError. - - :arg command: A list of commandline args - :arg input: Data to pipe to stdin. Omit for none. - - Remaining args have the same meaning as for Popen. - - """ - process = Popen(command, - stdout=PIPE, - stdin=PIPE, - stderr=PIPE, - shell=shell, - env=env) - out, err = process.communicate(input=input) - status = process.poll() # same as in check_output(), though wait() sounds better - if status: - error = CalledProcessError(status, command) - error.output = out - print('stdout output was:') - print(out) - print('stderr output was:') - print(err) - raise error - return out, err - - -def signed(content, private_key_name='signing.key'): - """Return the signed SHA-256 hash of ``content``, using the given key file.""" - command = ['openssl', 'dgst', '-sha256', '-sign', - join(tests_dir(), private_key_name)] - out, err = out_and_err(command, input=content) - return out - - -def install_le_auto(contents, install_path): - """Install some given source code as the letsencrypt-auto script at the - root level of a virtualenv. - - :arg contents: The contents of the built letsencrypt-auto script - :arg install_path: The path where to install the script - - """ - with open(install_path, 'w') as le_auto: - le_auto.write(contents) - chmod(install_path, S_IRUSR | S_IXUSR) - - -def run_le_auto(le_auto_path, venv_dir, base_url=None, le_auto_args_str='--version', **kwargs): - """Run the prebuilt version of letsencrypt-auto, returning stdout and - stderr strings. - - If the command returns other than 0, raise CalledProcessError. - - """ - env = environ.copy() - d = dict(VENV_PATH=venv_dir, - NO_CERT_VERIFY='1', - **kwargs) - - if base_url is not None: - # URL to PyPI-style JSON that tell us the latest released version - # of LE: - d['LE_AUTO_JSON_URL'] = base_url + 'certbot/json' - # URL to dir containing letsencrypt-auto and letsencrypt-auto.sig: - d['LE_AUTO_DIR_TEMPLATE'] = base_url + '%s/' - # The public key corresponding to signing.key: - d['LE_AUTO_PUBLIC_KEY'] = """-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg -tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G -hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT -uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl -LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 -Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 -iQIDAQAB ------END PUBLIC KEY-----""" - - env.update(d) - - return out_and_err( - le_auto_path + ' ' + le_auto_args_str, - shell=True, - env=env) - - -def set_le_script_version(venv_dir, version): - """Tell the letsencrypt script to report a certain version. - - We actually replace the script with a dummy version that knows only how to - print its version. - - """ - letsencrypt_path = join(venv_dir, 'bin', 'letsencrypt') - with open(letsencrypt_path, 'w') as script: - script.write("#!/usr/bin/env python\n" - "from sys import stderr\n" - "stderr.write('letsencrypt %s\\n')" % version) - chmod(letsencrypt_path, S_IRUSR | S_IXUSR) - - -def sudo_chmod(path, mode): - """Runs `sudo chmod mode path`.""" - mode = oct(mode).replace('o', '') - out_and_err(['sudo', 'chmod', mode, path]) - - -class AutoTests(TestCase): - """Test the major branch points of letsencrypt-auto: - - * An le-auto upgrade is needed. - * An le-auto upgrade is not needed. - * There was an out-of-date LE script installed. - * There was a current LE script installed. - * There was no LE script installed (less important). - * Pip hash-verification passes. - * Pip has a hash mismatch. - * The OpenSSL sig matches. - * The OpenSSL sig mismatches. - - For tests which get to the end, we run merely ``letsencrypt --version``. - The functioning of the rest of the certbot script is covered by other - test suites. - - """ - NEW_LE_AUTO = build_le_auto( - version='99.9.9', - requirements='letsencrypt==99.9.9 --hash=sha256:1cc14d61ab424cdee446f51e50f1123f8482ec740587fe78626c933bba2873a0') - NEW_LE_AUTO_SIG = signed(NEW_LE_AUTO) - - def test_successes(self): - """Exercise most branches of letsencrypt-auto. - - They just happen to be the branches in which everything goes well. - - I violate my usual rule of having small, decoupled tests, because... - - 1. We shouldn't need to run a Cartesian product of the branches: the - phases run in separate shell processes, containing state leakage - pretty effectively. The only shared state is FS state, and it's - limited to a temp dir, assuming (if we dare) all functions properly. - 2. One combination of branches happens to set us up nicely for testing - the next, saving code. - - """ - with temp_paths() as (le_auto_path, venv_dir): - # This serves a PyPI page with a higher version, a GitHub-alike - # with a corresponding le-auto script, and a matching signature. - resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, - 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} - with serving(resources) as base_url: - run_letsencrypt_auto = partial( - run_le_auto, - le_auto_path, - venv_dir, - base_url, - PIP_FIND_LINKS=join(tests_dir(), - 'fake-letsencrypt', - 'dist')) - - # Test when a phase-1 upgrade is needed, there's no LE binary - # installed, and pip hashes verify: - install_le_auto(build_le_auto(version='50.0.0'), le_auto_path) - out, err = run_letsencrypt_auto() - self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', - err.strip().splitlines()[-1])) - # Make a few assertions to test the validity of the next tests: - self.assertTrue('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) - - # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This - # conveniently sets us up to test the next 2 cases. - - # Test when neither phase-1 upgrade nor phase-2 upgrade is - # needed (probably a common case): - out, err = run_letsencrypt_auto() - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertFalse('Creating virtual environment...' in out) - - def test_phase2_upgrade(self): - """Test a phase-2 upgrade without a phase-1 upgrade.""" - resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, - 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} - with serving(resources) as base_url: - pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') - with temp_paths() as (le_auto_path, venv_dir): - install_le_auto(self.NEW_LE_AUTO, le_auto_path) - - # Create venv saving the correct bootstrap script version - out, err = run_le_auto(le_auto_path, venv_dir, base_url, - PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) - with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: - bootstrap_version = f.read() - - # Create a new venv with an old letsencrypt version - with temp_paths() as (le_auto_path, venv_dir): - venv_bin = join(venv_dir, 'bin') - makedirs(venv_bin) - set_le_script_version(venv_dir, '0.0.1') - with open(join(venv_dir, BOOTSTRAP_FILENAME), 'w') as f: - f.write(bootstrap_version) - - install_le_auto(self.NEW_LE_AUTO, le_auto_path) - out, err = run_le_auto(le_auto_path, venv_dir, base_url, - PIP_FIND_LINKS=pip_find_links) - - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) - - def test_openssl_failure(self): - """Make sure we stop if the openssl signature check fails.""" - with temp_paths() as (le_auto_path, venv_dir): - # Serve an unrelated hash signed with the good key (easier than - # making a bad key, and a mismatch is a mismatch): - resources = {'': 'certbot/', - 'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': build_le_auto(version='99.9.9'), - 'v99.9.9/letsencrypt-auto.sig': signed('something else')} - with serving(resources) as base_url: - copy_stable(LE_AUTO_PATH, le_auto_path) - try: - out, err = run_le_auto(le_auto_path, venv_dir, base_url) - except CalledProcessError as exc: - self.assertEqual(exc.returncode, 1) - self.assertTrue("Couldn't verify signature of downloaded " - "certbot-auto." in exc.output) - else: - print(out) - self.fail('Signature check on certbot-auto erroneously passed.') - - def test_pip_failure(self): - """Make sure pip stops us if there is a hash mismatch.""" - with temp_paths() as (le_auto_path, venv_dir): - resources = {'': 'certbot/', - 'certbot/json': dumps({'releases': {'99.9.9': None}})} - with serving(resources) as base_url: - # Build a le-auto script embedding a bad requirements file: - install_le_auto( - build_le_auto( - version='99.9.9', - requirements='configobj==5.0.6 --hash=sha256:badbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadb'), - le_auto_path) - try: - out, err = run_le_auto(le_auto_path, venv_dir, base_url) - except CalledProcessError as exc: - self.assertEqual(exc.returncode, 1) - self.assertTrue("THESE PACKAGES DO NOT MATCH THE HASHES " - "FROM THE REQUIREMENTS FILE" in exc.output) - self.assertFalse( - exists(venv_dir), - msg="The virtualenv was left around, even though " - "installation didn't succeed. We shouldn't do " - "this, as it foils our detection of whether we " - "need to recreate the virtualenv, which hinges " - "on the presence of $VENV_BIN/letsencrypt.") - else: - self.fail("Pip didn't detect a bad hash and stop the " - "installation.") - - def test_permissions_warnings(self): - """Make sure letsencrypt-auto properly warns about permissions problems.""" - # This test assumes that only the parent of the directory containing - # letsencrypt-auto (usually /tmp) may have permissions letsencrypt-auto - # considers insecure. - with temp_paths() as (le_auto_path, venv_dir): - le_auto_path = abspath(le_auto_path) - le_auto_dir = dirname(le_auto_path) - le_auto_dir_parent = dirname(le_auto_dir) - install_le_auto(self.NEW_LE_AUTO, le_auto_path) - - run_letsencrypt_auto = partial( - run_le_auto, le_auto_path, venv_dir, - le_auto_args_str='--install-only --no-self-upgrade', - PIP_FIND_LINKS=join(tests_dir(), 'fake-letsencrypt', 'dist')) - # Run letsencrypt-auto once with current permissions to avoid - # potential problems when the script tries to write to temporary - # directories. - run_letsencrypt_auto() - - le_auto_dir_mode = stat(le_auto_dir).st_mode - le_auto_dir_parent_mode = S_IMODE(stat(le_auto_dir_parent).st_mode) - try: - # Make letsencrypt-auto happy with the current permissions - chmod(le_auto_dir, S_IRUSR | S_IXUSR) - sudo_chmod(le_auto_dir_parent, 0o755) - - self._test_permissions_warnings_about_path(le_auto_path, run_letsencrypt_auto) - self._test_permissions_warnings_about_path(le_auto_dir, run_letsencrypt_auto) - finally: - chmod(le_auto_dir, le_auto_dir_mode) - sudo_chmod(le_auto_dir_parent, le_auto_dir_parent_mode) - - def _test_permissions_warnings_about_path(self, path, run_le_auto_func): - # Test that there are no problems with the current permissions - out, _ = run_le_auto_func() - self.assertFalse('insecure permissions' in out) - - stat_result = stat(path) - original_mode = stat_result.st_mode - - # Test world permissions - chmod(path, original_mode | S_IWOTH) - out, _ = run_le_auto_func() - self.assertTrue('insecure permissions' in out) - - # Test group permissions - if stat_result.st_gid >= 1000: - chmod(path, original_mode | S_IWGRP) - out, _ = run_le_auto_func() - self.assertTrue('insecure permissions' in out) - - # Test owner permissions - if stat_result.st_uid >= 1000: - chmod(path, original_mode | S_IWUSR) - out, _ = run_le_auto_func() - self.assertTrue('insecure permissions' in out) - - # Test that permissions were properly restored - chmod(path, original_mode) - out, _ = run_le_auto_func() - self.assertFalse('insecure permissions' in out) - - def test_disabled_permissions_warnings(self): - """Make sure that letsencrypt-auto permissions warnings can be disabled.""" - with temp_paths() as (le_auto_path, venv_dir): - le_auto_path = abspath(le_auto_path) - install_le_auto(self.NEW_LE_AUTO, le_auto_path) - - le_auto_args_str='--install-only --no-self-upgrade' - pip_links=join(tests_dir(), 'fake-letsencrypt', 'dist') - out, _ = run_le_auto(le_auto_path, venv_dir, - le_auto_args_str=le_auto_args_str, - PIP_FIND_LINKS=pip_links) - self.assertTrue('insecure permissions' in out) - - # Test that warnings are disabled when the script isn't run as - # root. - out, _ = run_le_auto(le_auto_path, venv_dir, - le_auto_args_str=le_auto_args_str, - LE_AUTO_SUDO='', - PIP_FIND_LINKS=pip_links) - self.assertFalse('insecure permissions' in out) - - # Test that --no-permissions-check disables warnings. - le_auto_args_str += ' --no-permissions-check' - out, _ = run_le_auto( - le_auto_path, venv_dir, - le_auto_args_str=le_auto_args_str, - PIP_FIND_LINKS=pip_links) - self.assertFalse('insecure permissions' in out) diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh deleted file mode 100644 index 8bdffec87..000000000 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash -set -e -# Start by making sure your system is up-to-date: -yum update -y >/dev/null -yum install -y centos-release-scl >/dev/null -yum install -y python27 >/dev/null 2>/dev/null - -LE_AUTO_PY_34="certbot/letsencrypt-auto-source/letsencrypt-auto_py_34" -LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto" - -# Last version of certbot-auto that was bootstraping Python 3.4 for CentOS 6 users -INITIAL_CERTBOT_VERSION_PY34="certbot 0.38.0" - -# we're going to modify env variables, so do this in a subshell -( -# ensure CentOS6 32bits is not supported anymore, and so certbot is not installed -export UNAME_FAKE_32BITS=true -if ! "$LE_AUTO" 2>&1 | grep -q "Certbot cannot be installed."; then - echo "ERROR: certbot-auto installed certbot on 32-bit CentOS." - exit 1 -fi -) - -echo "PASSED: On CentOS 6 32 bits, certbot-auto refused to install certbot." - -# we're going to modify env variables, so do this in a subshell -( - . /opt/rh/python27/enable - - # ensure python 3 isn't installed - if python3 --version 2> /dev/null; then - echo "ERROR: Python3 is already installed." - exit 1 - fi - - # ensure python2.7 is available - if ! python2.7 --version 2> /dev/null; then - echo "ERROR: Python2.7 is not available." - exit 1 - fi - - # bootstrap, but don't install python 3. - "$LE_AUTO" --no-self-upgrade -n --version > /dev/null 2> /dev/null - - # ensure python 3 isn't installed - if python3 --version 2> /dev/null; then - echo "ERROR: letsencrypt-auto installed Python3 even though Python2.7 is present." - exit 1 - fi - - echo "PASSED: Did not upgrade to Python3 when Python2.7 is present." -) - -# ensure python2.7 isn't available -if python2.7 --version 2> /dev/null; then - echo "ERROR: Python2.7 is still available." - exit 1 -fi - -# Skip self upgrade due to Python 3 not being available. -if ! "$LE_AUTO" 2>&1 | grep -q "WARNING: couldn't find Python"; then - echo "ERROR: Python upgrade failure warning not printed!" - exit 1 -fi - -# bootstrap from the old letsencrypt-auto, this time installing python3.4 -"$LE_AUTO_PY_34" --no-self-upgrade -n --version >/dev/null 2>/dev/null - -# ensure python 3.4 is installed -if ! python3.4 --version >/dev/null 2>/dev/null; then - echo "ERROR: letsencrypt-auto failed to install Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present." - exit 1 -fi - -echo "PASSED: Successfully upgraded to Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present." - -# As "certbot-auto" (so without implicit --non-interactive flag set), check that the script -# refuses to install SCL Python 3.6 when run in a non interactive shell (simulated here -# using | tee /dev/null) if --non-interactive flag is not provided. -cp "$LE_AUTO" /tmp/certbot-auto -# NB: Readline has an issue on all Python versions for CentOS 6, making `certbot --version` -# output an unprintable ASCII character on a new line at the end. -# So we take the second last line of the output. -version=$(/tmp/certbot-auto --version 2>/dev/null | tee /dev/null | tail -2 | head -1) - -if [ "$version" != "$INITIAL_CERTBOT_VERSION_PY34" ]; then - echo "ERROR: certbot-auto upgraded certbot in a non-interactive shell with --non-interactive flag not set." - exit 1 -fi - -echo "PASSED: certbot-auto did not upgrade certbot in a non-interactive shell with --non-interactive flag not set." - -if [ -f /opt/rh/rh-python36/enable ]; then - echo "ERROR: certbot-auto installed Python3.6 in a non-interactive shell with --non-interactive flag not set." - exit 1 -fi - -echo "PASSED: certbot-auto did not install Python3.6 in a non-interactive shell with --non-interactive flag not set." - -# now bootstrap from current letsencrypt-auto, that will install python3.6 from SCL -"$LE_AUTO" --no-self-upgrade -n --version >/dev/null 2>/dev/null - -# Following test is executed in a subshell, to not leak any environment variable -( - # enable SCL rh-python36 - . /opt/rh/rh-python36/enable - - # ensure python 3.6 is installed - if ! python3.6 --version >/dev/null 2>/dev/null; then - echo "ERROR: letsencrypt-auto failed to install Python3.6 using current letsencrypt-auto when only Python2.6/Python3.4 are present." - exit 1 - fi - - echo "PASSED: Successfully upgraded to Python3.6 using current letsencrypt-auto when only Python2.6/Python3.4 are present." -) - -# Following test is executed in a subshell, to not leak any environment variable -( - export VENV_PATH=$(mktemp -d) - "$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1 - if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1-2)" != "3.6" ]; then - echo "ERROR: Python 3.6 wasn't used with --no-bootstrap!" - exit 1 - fi -) - -# Following test is executed in a subshell, to not leak any environment variable -( - # enable SCL rh-python36 - . /opt/rh/rh-python36/enable - - # ensure everything works fine with certbot-auto bootstrap when python 3.6 is already enabled - export VENV_PATH=$(mktemp -d) - if ! "$LE_AUTO" --no-self-upgrade -n --version >/dev/null 2>/dev/null; then - echo "ERROR: Certbot-auto broke when Python 3.6 SCL is already enabled." - exit 1 - fi -) - -# we're going to modify env variables, so do this in a subshell -( - # ensure CentOS6 32bits is not supported anymore, and so certbot - # is not upgraded nor reinstalled. - export UNAME_FAKE_32BITS=true - OUTPUT=$("$LE_AUTO" --version 2>&1) - if ! echo "$OUTPUT" | grep -q "Certbot will no longer receive updates."; then - echo "ERROR: certbot-auto failed to run or upgraded pre-existing Certbot instance on 32-bit CentOS 6." - exit 1 - fi - if ! "$LE_AUTO" --install-only 2>&1 | grep -q "Certbot cannot be installed."; then - echo "ERROR: certbot-auto reinstalled Certbot on 32-bit CentOS 6." - exit 1 - fi -) - -# we're going to modify env variables, so do this in a subshell -( - # Prepare a certbot installation in the old venv path - rm -rf /opt/eff.org - VENV_PATH=~/.local/share/letsencrypt "$LE_AUTO" --install-only > /dev/null 2> /dev/null - # fake 32 bits mode - export UNAME_FAKE_32BITS=true - OUTPUT=$("$LE_AUTO" --version 2>&1) - if ! echo "$OUTPUT" | grep -q "Certbot will no longer receive updates."; then - echo "ERROR: certbot-auto failed to run or upgraded pre-existing Certbot instance in the old venv path on 32-bit CentOS 6." - exit 1 - fi -) - -echo "PASSED: certbot-auto refused to install/upgrade certbot on 32-bit CentOS 6." - -# test using python3 -pytest -v -s certbot/letsencrypt-auto-source/tests diff --git a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem deleted file mode 100644 index 4e4d29bd2..000000000 --- a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem +++ /dev/null @@ -1,23 +0,0 @@ ------BEGIN CERTIFICATE----- -MIID5jCCAs6gAwIBAgIJAI1Qkfyw88REMA0GCSqGSIb3DQEBBQUAMFUxCzAJBgNV -BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMRswGQYDVQQKExJNeSBCb2d1cyBS -b290IENlcnQxFDASBgNVBAMTC2V4YW1wbGUuY29tMB4XDTE1MTIwNDIwNTIxNVoX -DTQwMTIwMzIwNTIxNVowVTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3Rh -dGUxGzAZBgNVBAoTEk15IEJvZ3VzIFJvb3QgQ2VydDEUMBIGA1UEAxMLZXhhbXBs -ZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQVQpQ2EH4gTJB -NJP6+ocT3xJwT8mSXYUnvzjj6iv+JxZiXRGzAPziNzrrSRKY0yDHF+UiJwuOerLa -n8laZkLb1Ogqzs2u64rKeb0xWv90Qp+eXG0J/1xb4dw+GExqe5QFo1JUJzO/eK7m -1S04SeFkN1qV9mD5yJUy7DGiTUzDHgCxM2tXMLusXYqkxsQQ9+2EJ7BEOK4YJGEx -Sign5FuSxb64PiNow6OA97CaLl7tV4INP4w195ueDRIaS4poeOep4s8U7IAdMjIZ -EryJgKNCij50xK92vPBBJSj0NOitltBlwoEqkOZpQCOZamFd6nvt78LQ6W8Am+l6 -y6oCON5JAgMBAAGjgbgwgbUwHQYDVR0OBBYEFAlrdStDhaayLLj89Whe3Gc+HE8y -MIGFBgNVHSMEfjB8gBQJa3UrQ4Wmsiy4/PVoXtxnPhxPMqFZpFcwVTELMAkGA1UE -BhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxGzAZBgNVBAoTEk15IEJvZ3VzIFJv -b3QgQ2VydDEUMBIGA1UEAxMLZXhhbXBsZS5jb22CCQCNUJH8sPPERDAMBgNVHRME -BTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQC7KAQfDTiNM3QO8Ic3x21CAPJUavkH -zshifN+Ei0+nmseHDTCTgsGfGDOToLUpUEZ4PuiHnz08UwRfd9wotc3SgY9ZaXMe -vRs8KUAF9EoyTvESzPyv2b6cS9NNMpj5y7KyXSyP17VoGbNavtiGQ4dwgEH6VgNl -0RtBvcSBv/tqxIIx1tWzL74tVEm0Kbd9BAZsYpQNKL8e6WXP35/j0PvCCvtofGrA -E8LTqMz4kCwnX+QaJIMJhBophRCsjXdAkvFbFxX0DGPztQtzIwBPcdMjsft7AFeE -0XchhDDXxw8YsbpvPfCvrD8XiiVuBycbnB1zt0LLVwB/QsCzUW9ImpLC ------END CERTIFICATE----- diff --git a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem deleted file mode 100644 index 9caa7ddaa..000000000 --- a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA0FUKUNhB+IEyQTST+vqHE98ScE/Jkl2FJ7844+or/icWYl0R -swD84jc660kSmNMgxxflIicLjnqy2p/JWmZC29ToKs7NruuKynm9MVr/dEKfnlxt -Cf9cW+HcPhhManuUBaNSVCczv3iu5tUtOEnhZDdalfZg+ciVMuwxok1Mwx4AsTNr -VzC7rF2KpMbEEPfthCewRDiuGCRhMUooJ+RbksW+uD4jaMOjgPewmi5e7VeCDT+M -Nfebng0SGkuKaHjnqeLPFOyAHTIyGRK8iYCjQoo+dMSvdrzwQSUo9DTorZbQZcKB -KpDmaUAjmWphXep77e/C0OlvAJvpesuqAjjeSQIDAQABAoIBAH+qbVzneV3wxjwh -HUHi/p3VyHXc3xh7iNq3mwRH/1eK2nPCttLsGwwBbnC64dOXJfH7maWZKcLRPAMv -gfOM0RHn4bJB8tdrbizv91lke0DihvBDkWpb+1wvB4lh2Io0Wpwt3ojFUTfXm87G -+iQRWjbQmQlm5zyKh6uiBDSCjDTQdb9omZEBMAwlGPTZwt8TRUEtWd8QgW8FCHoB -iLER2WBwXdvn3PBtocI3VE6IYDSeZ81Xv+d7925RtVintT8Suk4toYwX+jfSz+wZ -sgHd5V6PSv9a7GUlWoUihD99D9wqDZE8IvMDZ5ofSAUd1KfICDtmsEyugY7u2yYZ -tYt49AECgYEA73f7ITMHg8JsUipqb6eG10gCRtRhkqrrO1g/TNeTBh3CTrQGb56e -y6kmUivn5gK46t3T2N4Ht4IR8fpLcJcbPYPQNulSjmWm5y6WduafXW/VCW1NA9Lc -FyGPkMxFCIVJTLFxfLFepBVvtUzLLDKGGtQxru/GNbBzjdtmVfDPIoECgYEA3rbM -cTfvj+jWrV1YsRbphyjy+k3OJEIVx6KA4s5d7Tp12UfYQp/B3HPhXXm5wqeo1Nos -UAEWZIMi1VoE8iu6jjeJ6uERtbKKQVed25Us/ff0jUPbxlXgiBOtRcllq9d9Srjm -ybHUgfjLsZ2/xpIcOl+oI5pDM9JvD8Sq4ZCFR8kCgYBK/H0tFjeiML2OtS2DLShy -PWBJIbQ0I0Vp3eZkf5TQc30m/ASP61G6YItZa9pAElYpZbEy1cQA2MAZz9DTvt2O -07ndmA57/KTY+6OuM+Vvctd5DjrxmZPFwoKcSvrLAkHDvETXUQtbwkKquRNeEawg -tpWgPAELSufEYhGXk8KpAQKBgBDCqPgMQZcO6rj5QWdyVfi5+C8mE9Fet8ziSdjH -twHXWG8VnQzGgQxaHCewtW4Ut/vsv1D2A/1kcQalU6H18IArZdGrRm3qFcV9FoAj -5dLnChxncu6mH9Odx3htA52/BcrNx3B+VYPCeXHQcVI8RKuP71NelJgdygXhwwpe -mekhAoGBAOUovnqylciYa9HRqo+xZk59eyX+ehhnlV8SeJ2K0PwaQkzQ0KYtCmE7 -kdSdhcv8h/IQKGaFfc/LyFMM/a26PfAeY5bj41UjkT0K5hQrYuL/52xaT401YLcb -Xo+bZz9K0hrdP7TdZFuTY/WxojXgjsVAuAN1NwnJumqxhzPh+hfl ------END RSA PRIVATE KEY----- diff --git a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl deleted file mode 100644 index ad6d262b4..000000000 --- a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl +++ /dev/null @@ -1 +0,0 @@ -D613482D0EF95DD0 diff --git a/letsencrypt-auto-source/tests/certs/localhost/cert.pem b/letsencrypt-auto-source/tests/certs/localhost/cert.pem deleted file mode 100644 index ac83535ce..000000000 --- a/letsencrypt-auto-source/tests/certs/localhost/cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDKjCCAhICCQDWE0gtDvld0DANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJB -VTETMBEGA1UECBMKU29tZS1TdGF0ZTEbMBkGA1UEChMSTXkgQm9ndXMgUm9vdCBD -ZXJ0MRQwEgYDVQQDEwtleGFtcGxlLmNvbTAeFw0xNTEyMDQyMDU0MzFaFw00MDEy -MDMyMDU0MzFaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEw -HwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCWxvY2Fs -aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2WIIi86Mis4UQH -a5PrFbX2PBtQHbI3t3ekN1CewRsgQ/2X3lCeWhKmr3CJYXVgA7q/23PORQAiuV6y -DG2dQIrjeahWCXaCptTi49ljfVRTW2IxrHke/iA8TkDuZbWGzVLb8TB83ipBOD41 -SjuomoN4A/ktnIfbNqRqgjjHs2wwJHDfxPiCQlwyOayjHmdlh8cqfVE8rWEm5/3T -Iu0X1J53SammR1SbUmsLJNofxFYMK1ogHb0CaFEG9QuuUDPJl5K74Rr6InMQZKPn -ne4W3cGoALxPHAca7yicpSMSmdsmd6pqylc2Fdua7o/wf0SwShxS4A1DqA/HWLEM -V6MSEF8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAz5sMAFG6W/ZEULZITkBTCU6P -NttpGiKufnqyBW5HyNylaczfnHnClvQjr8f/84xvKVcfC3xP0lz+92aIQqo+5L/n -v7gLhBFR4Vr2XwMt2qz2FpkaxmVwnhVAHaaC05WIKQ6W2gDwWT0u1K8YdTh+7mvN -AT9FW4vDgtNZWq4W/PePh9QCiOOQhGOuBYj/7zqLtz4XPifhi66ILIRDHiu0kond -3YMFcECIAf4MPT9vT0iNcWX+c8CfAixPt8nMD6bzOo3oTcfuZh/2enfgLbMqOlOi -uk72FM5VVPXTWAckJvL/vVjqsvDuJQKqbr0oUc3bdWbS36xtWZUycp4IQLguAQ== ------END CERTIFICATE----- diff --git a/letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem b/letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem deleted file mode 100644 index 8a6189f88..000000000 --- a/letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICnjCCAYYCAQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUx -ITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAxMJbG9j -YWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZYgiLzoyKzh -RAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/ZfeUJ5aEqavcIlhdWADur/bc85FACK5 -XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGseR7+IDxOQO5ltYbNUtvxMHzeKkE4 -PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E+IJCXDI5rKMeZ2WHxyp9UTytYSbn -/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAdvQJoUQb1C65QM8mXkrvhGvoicxBk -o+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrKVzYV25ruj/B/RLBKHFLgDUOoD8dY -sQxXoxIQXwIDAQABoAAwDQYJKoZIhvcNAQEFBQADggEBAFbg3WrAokoPx7iAYG6z -PqeDd4/XanXjeL4Ryxv6LoGhu69mmBAd3N5ILPyQJjnkWpIjEmJDzEcPMzhQjRh5 -GlWTyvKWO4zClYU840KZk7crVkpzNZ+HP0YeM/Agz6sab00ffRcq5m1wEF9MCvDE -8FUXk1HBHRAb/6t9QV/7axsPOkGT8SjQ1v2SCaiB0HQL3sYChYLi5zu4dfmQNPGq -ar9Xm5a0YqOQIFfmy8RSwxk0Q/ipNFTGN1uvlIRkgbT9zPnodxjWZsSI9BF+q5Af -uiE/oAk7MxfJ0LyLfhOWB+T98bKIOVtFT3wMLS1IIgMogwqCEXFf30Q9p2iTEzqT -6UE= ------END CERTIFICATE REQUEST----- diff --git a/letsencrypt-auto-source/tests/certs/localhost/privkey.pem b/letsencrypt-auto-source/tests/certs/localhost/privkey.pem deleted file mode 100644 index 18feba403..000000000 --- a/letsencrypt-auto-source/tests/certs/localhost/privkey.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEArZYgiLzoyKzhRAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/Zfe -UJ5aEqavcIlhdWADur/bc85FACK5XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGs -eR7+IDxOQO5ltYbNUtvxMHzeKkE4PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E -+IJCXDI5rKMeZ2WHxyp9UTytYSbn/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAd -vQJoUQb1C65QM8mXkrvhGvoicxBko+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrK -VzYV25ruj/B/RLBKHFLgDUOoD8dYsQxXoxIQXwIDAQABAoIBAG8bVJ+xKt6nqVg9 -16HKKw9ZGIfy888K0qgFuFImCzwtntdGycmYUdb2Uf0aMgNK/ZgfDXxGXuwDTdtK -46GVsaY0i74vs8bjQZ2pzGVsxN+gqzFi0h6Es+w2LXBqJzfVnL6YgPykMB+jtzg6 -K9Wbyaq0uvZXN4XNzl/WvJtTV4i7Cff1MOd5EhKFdqxrZvB/SRBCr/SMMafRtB9P -EvMneNKzhmlrutHAxuyxEKZR32Kkx7ydAdTjGgn+rE+NL5BweXfeWhLU4Bv14bn9 -Mkneu3w5o1ryJfE2YnVajUP//jeopUT0nTQ3MpEusBQCLBlvFXjjM9uCaFX+5+MP -0H4xVcECgYEA1Q+wR3GHbk37vIGSlbENyUsri5WlMt8IVAHsDsTOpxAjYB0yyo+x -h9RS+RJZQECJlA6H72peUl3GM7RgdWIcKOT3nZ12XqYKG57rr/N5zlUuxbdS8KBk -JhyZeJdYjq/Jrno1ZP+OSmc7VvBLcM7irY7LHlvK0o8W1W0TNJ8jrZkCgYEA0JHX -lJd+fiezcUS7g4moHtzJp0JKquQiXLX+c2urmpyhb3ZrTuQ8OUjSy6DlwHlgDx8K -Hg2sdx/ZCuDaGjR4IY/Qs5RFt9WUqlK9gi9V3nYVrzBOQkdFOf/Ad3j4pQ8/aeCX -nP6snHXz1WqPpbCXG6l6GzFGbQU473GfuKsDuLcCgYAWQaNKc0OQdDj9whNL68ji -5CVSWXl+TOoTzHeaO1jS/s6TNbmei1AiPj3EovQL0DIO802j5tqfhAg2UntZB7yl -UPXE0zQQQwv/QqSgJrDsqt1N7g6N8FNF3+rwO+8WSKqqvT1ipYd5ojsCo+tdh18K -fkYdj70qLaRW+yPsdUtG0QKBgEYc8NqbvsML94+ZKmwCh4iwcf2PFGi0PjTqXTpR -tKNKCh7dMR+ZLAGZ0HrxgKqeYsNSjOUjdZmqFB1LDyaGAuhNXzwvGOy+mLZVEC3G -Wdhp28pDs9sl+EiSCBJhkTxzjr656F23YzFJmYlhxB5P6cw7wbeIbgNSIRylFqtO -mfarAoGBAICsAEWypOctxtmtOcjxgJ7jMbOA7rrsGlXpiy1/WlwIwRGF5LMvIIFX -qFAfiPcZn05ZgdAGzaFYowdjmQB10FW0jZbDf+nIHfOF5YmfmfWjsaweEGALJmqB -okGu/lGNGf3XoYzy0/hC3WAqk3znSZtQLUq8jEWF7dLNUizUeUow ------END RSA PRIVATE KEY----- diff --git a/letsencrypt-auto-source/tests/certs/localhost/server.pem b/letsencrypt-auto-source/tests/certs/localhost/server.pem deleted file mode 100644 index c5765dd89..000000000 --- a/letsencrypt-auto-source/tests/certs/localhost/server.pem +++ /dev/null @@ -1,46 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEArZYgiLzoyKzhRAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/Zfe -UJ5aEqavcIlhdWADur/bc85FACK5XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGs -eR7+IDxOQO5ltYbNUtvxMHzeKkE4PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E -+IJCXDI5rKMeZ2WHxyp9UTytYSbn/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAd -vQJoUQb1C65QM8mXkrvhGvoicxBko+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrK -VzYV25ruj/B/RLBKHFLgDUOoD8dYsQxXoxIQXwIDAQABAoIBAG8bVJ+xKt6nqVg9 -16HKKw9ZGIfy888K0qgFuFImCzwtntdGycmYUdb2Uf0aMgNK/ZgfDXxGXuwDTdtK -46GVsaY0i74vs8bjQZ2pzGVsxN+gqzFi0h6Es+w2LXBqJzfVnL6YgPykMB+jtzg6 -K9Wbyaq0uvZXN4XNzl/WvJtTV4i7Cff1MOd5EhKFdqxrZvB/SRBCr/SMMafRtB9P -EvMneNKzhmlrutHAxuyxEKZR32Kkx7ydAdTjGgn+rE+NL5BweXfeWhLU4Bv14bn9 -Mkneu3w5o1ryJfE2YnVajUP//jeopUT0nTQ3MpEusBQCLBlvFXjjM9uCaFX+5+MP -0H4xVcECgYEA1Q+wR3GHbk37vIGSlbENyUsri5WlMt8IVAHsDsTOpxAjYB0yyo+x -h9RS+RJZQECJlA6H72peUl3GM7RgdWIcKOT3nZ12XqYKG57rr/N5zlUuxbdS8KBk -JhyZeJdYjq/Jrno1ZP+OSmc7VvBLcM7irY7LHlvK0o8W1W0TNJ8jrZkCgYEA0JHX -lJd+fiezcUS7g4moHtzJp0JKquQiXLX+c2urmpyhb3ZrTuQ8OUjSy6DlwHlgDx8K -Hg2sdx/ZCuDaGjR4IY/Qs5RFt9WUqlK9gi9V3nYVrzBOQkdFOf/Ad3j4pQ8/aeCX -nP6snHXz1WqPpbCXG6l6GzFGbQU473GfuKsDuLcCgYAWQaNKc0OQdDj9whNL68ji -5CVSWXl+TOoTzHeaO1jS/s6TNbmei1AiPj3EovQL0DIO802j5tqfhAg2UntZB7yl -UPXE0zQQQwv/QqSgJrDsqt1N7g6N8FNF3+rwO+8WSKqqvT1ipYd5ojsCo+tdh18K -fkYdj70qLaRW+yPsdUtG0QKBgEYc8NqbvsML94+ZKmwCh4iwcf2PFGi0PjTqXTpR -tKNKCh7dMR+ZLAGZ0HrxgKqeYsNSjOUjdZmqFB1LDyaGAuhNXzwvGOy+mLZVEC3G -Wdhp28pDs9sl+EiSCBJhkTxzjr656F23YzFJmYlhxB5P6cw7wbeIbgNSIRylFqtO -mfarAoGBAICsAEWypOctxtmtOcjxgJ7jMbOA7rrsGlXpiy1/WlwIwRGF5LMvIIFX -qFAfiPcZn05ZgdAGzaFYowdjmQB10FW0jZbDf+nIHfOF5YmfmfWjsaweEGALJmqB -okGu/lGNGf3XoYzy0/hC3WAqk3znSZtQLUq8jEWF7dLNUizUeUow ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDKjCCAhICCQDWE0gtDvld0DANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJB -VTETMBEGA1UECBMKU29tZS1TdGF0ZTEbMBkGA1UEChMSTXkgQm9ndXMgUm9vdCBD -ZXJ0MRQwEgYDVQQDEwtleGFtcGxlLmNvbTAeFw0xNTEyMDQyMDU0MzFaFw00MDEy -MDMyMDU0MzFaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEw -HwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCWxvY2Fs -aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2WIIi86Mis4UQH -a5PrFbX2PBtQHbI3t3ekN1CewRsgQ/2X3lCeWhKmr3CJYXVgA7q/23PORQAiuV6y -DG2dQIrjeahWCXaCptTi49ljfVRTW2IxrHke/iA8TkDuZbWGzVLb8TB83ipBOD41 -SjuomoN4A/ktnIfbNqRqgjjHs2wwJHDfxPiCQlwyOayjHmdlh8cqfVE8rWEm5/3T -Iu0X1J53SammR1SbUmsLJNofxFYMK1ogHb0CaFEG9QuuUDPJl5K74Rr6InMQZKPn -ne4W3cGoALxPHAca7yicpSMSmdsmd6pqylc2Fdua7o/wf0SwShxS4A1DqA/HWLEM -V6MSEF8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAz5sMAFG6W/ZEULZITkBTCU6P -NttpGiKufnqyBW5HyNylaczfnHnClvQjr8f/84xvKVcfC3xP0lz+92aIQqo+5L/n -v7gLhBFR4Vr2XwMt2qz2FpkaxmVwnhVAHaaC05WIKQ6W2gDwWT0u1K8YdTh+7mvN -AT9FW4vDgtNZWq4W/PePh9QCiOOQhGOuBYj/7zqLtz4XPifhi66ILIRDHiu0kond -3YMFcECIAf4MPT9vT0iNcWX+c8CfAixPt8nMD6bzOo3oTcfuZh/2enfgLbMqOlOi -uk72FM5VVPXTWAckJvL/vVjqsvDuJQKqbr0oUc3bdWbS36xtWZUycp4IQLguAQ== ------END CERTIFICATE----- diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz b/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz deleted file mode 100644 index 5f9a48a34..000000000 Binary files a/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz and /dev/null differ diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py b/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py deleted file mode 100755 index 9d811fab5..000000000 --- a/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py +++ /dev/null @@ -1,8 +0,0 @@ -from sys import argv, stderr - - -def main(): - """Act like letsencrypt --version insofar as printing the version number to - stderr.""" - if '--version' in argv: - stderr.write('letsencrypt 99.9.9\n') diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py b/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py deleted file mode 100644 index e5f7fde35..000000000 --- a/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import setup - - -setup( - name='letsencrypt', - version='99.9.9', - description='A mock version of letsencrypt that just prints its version', - py_modules=['letsencrypt'], - entry_points={ - 'console_scripts': ['letsencrypt = letsencrypt:main'] - } -) diff --git a/letsencrypt-auto-source/tests/oraclelinux6_tests.sh b/letsencrypt-auto-source/tests/oraclelinux6_tests.sh deleted file mode 100644 index f3fd952f3..000000000 --- a/letsencrypt-auto-source/tests/oraclelinux6_tests.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -set -eo pipefail -# Start by making sure your system is up-to-date: -yum update -y >/dev/null - -LE_AUTO_PY_34="certbot/letsencrypt-auto-source/letsencrypt-auto_py_34" -LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto" - -# Apply installation instructions from official documentation: -# https://certbot.eff.org/lets-encrypt/centosrhel6-other -cp "$LE_AUTO" /usr/local/bin/certbot-auto -chown root /usr/local/bin/certbot-auto -chmod 0755 /usr/local/bin/certbot-auto -LE_AUTO=/usr/local/bin/certbot-auto - -# Last version of certbot-auto that was bootstraping Python 3.4 for CentOS 6 users -INITIAL_CERTBOT_VERSION_PY34="certbot 0.38.0" - -# Check bootstrap from current certbot-auto will fail, because SCL is not enabled. -set +o pipefail -if ! "$LE_AUTO" -n 2>&1 | grep -q "Enable the SCL repository and try running Certbot again."; then - echo "ERROR: Bootstrap was not aborted although SCL was not installed!" - exit 1 -fi -set -o pipefail - -echo "PASSED: Bootstrap was aborted since SCL was not installed." - -# Bootstrap from the old letsencrypt-auto, Python 3.4 will be installed from EPEL. -"$LE_AUTO_PY_34" --no-self-upgrade -n --install-only >/dev/null 2>/dev/null - -# Ensure Python 3.4 is installed -if ! command -v python3.4 &>/dev/null; then - echo "ERROR: old letsencrypt-auto failed to install Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present." - exit 1 -fi - -echo "PASSED: Bootstrap from old letsencrypt-auto succeeded and installed Python 3.4" - -# Expect certbot-auto to skip rebootstrapping with a warning since SCL is not installed. -if ! "$LE_AUTO" --non-interactive --version 2>&1 | grep -q "This requires manual user intervention"; then - echo "FAILED: Script certbot-auto did not print a warning about needing manual intervention!" - exit 1 -fi - -echo "PASSED: Script certbot-auto did not rebootstrap." - -# NB: Readline has an issue on all Python versions for OL 6, making `certbot --version` -# output an unprintable ASCII character on a new line at the end. -# So we take the second last line of the output. -version=$($LE_AUTO --version 2>/dev/null | tail -2 | head -1) - -if [ "$version" != "$INITIAL_CERTBOT_VERSION_PY34" ]; then - echo "ERROR: Script certbot-auto upgraded certbot in a non-interactive shell while SCL was not enabled." - exit 1 -fi - -echo "PASSED: Script certbot-auto did not upgrade certbot but started it successfully while SCL was not enabled." - -# Enable SCL -yum install -y oracle-softwarecollection-release-el6 >/dev/null - -# Expect certbot-auto to bootstrap successfully since SCL is available. -"$LE_AUTO" -n --version &>/dev/null - -if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1-2)" != "3.6" ]; then - echo "ERROR: Script certbot-auto failed to bootstrap and install Python 3.6 while SCL is available." - exit 1 -fi - -if ! /opt/eff.org/certbot/venv/bin/certbot --version > /dev/null 2> /dev/null; then - echo "ERROR: Script certbot-auto did not install certbot correctly while SCL is enabled." - exit 1 -fi - -echo "PASSED: Script certbot-auto correctly bootstraped Certbot using rh-python36 when SCL is available." - -# Expect certbot-auto will be totally silent now that everything has been correctly boostraped. -OUTPUT_LEN=$("$LE_AUTO" --install-only --no-self-upgrade --quiet 2>&1 | wc -c) -if [ "$OUTPUT_LEN" != 0 ]; then - echo certbot-auto produced unexpected output! - exit 1 -fi - -echo "PASSED: Script certbot-auto did not print anything in quiet mode." diff --git a/letsencrypt-auto-source/tests/uname_wrapper.sh b/letsencrypt-auto-source/tests/uname_wrapper.sh deleted file mode 100644 index df1f568c6..000000000 --- a/letsencrypt-auto-source/tests/uname_wrapper.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e - -uname_output=$(/bin/uname_orig "$@") - -if [ "$UNAME_FAKE_32BITS" = true ]; then - uname_output="${uname_output//x86_64/i686}" -fi - -echo "$uname_output" diff --git a/linter_plugin.py b/linter_plugin.py index 75879f73a..a19bf7df9 100644 --- a/linter_plugin.py +++ b/linter_plugin.py @@ -10,7 +10,9 @@ from pylint.checkers import BaseChecker from pylint.interfaces import IAstroidChecker # Modules in theses packages can import the os module. -WHITELIST_PACKAGES = ['acme', 'certbot_compatibility_test', 'lock_test'] +WHITELIST_PACKAGES = [ + 'acme', 'certbot_integration_tests', 'certbot_compatibility_test', 'lock_test' +] class ForbidStandardOsModule(BaseChecker): @@ -25,8 +27,8 @@ class ForbidStandardOsModule(BaseChecker): 'E5001': ( 'Forbidden use of os module, certbot.compat.os must be used instead', 'os-module-forbidden', - 'Some methods from the standard os module cannot be used for security reasons on Windows: ' - 'the safe wrapper certbot.compat.os must be used instead in Certbot.' + 'Some methods from the standard os module cannot be used for security reasons on ' + 'Windows: the safe wrapper certbot.compat.os must be used instead in Certbot.' ) } priority = -1 diff --git a/pull_request_template.md b/pull_request_template.md index c806d33e8..53298291b 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -1,4 +1,5 @@ ## Pull Request Checklist - [ ] If the change being made is to a [distributed component](https://certbot.eff.org/docs/contributing.html#code-components-and-layout), edit the `master` section of `certbot/CHANGELOG.md` to include a description of the change being made. +- [ ] Add or update any documentation as needed to support the changes in this PR. - [ ] Include your name in `AUTHORS.md` if you like. diff --git a/pytest.ini b/pytest.ini index 16aa9a193..d7fe53494 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,8 +2,6 @@ # settings we want to also change there must be added to the release script # directly. [pytest] -# In general, all warnings are treated as errors. Here are the exceptions: -# 1- decodestring: https://github.com/rthalley/dnspython/issues/338 # Warnings being triggered by our plugins using deprecated features in # acme/certbot should be fixed by having our plugins no longer using the # deprecated code rather than adding them to the list of ignored warnings here. @@ -13,4 +11,3 @@ # we release breaking changes. filterwarnings = error - ignore:decodestring:DeprecationWarning diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 5fbf8503d..d53fba88b 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -20,13 +20,13 @@ adopt-info: certbot apps: certbot: - command: bin/python3 $SNAP/bin/certbot + command: bin/python3 -s $SNAP/bin/certbot environment: PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" AUGEAS_LENS_LIB: "$SNAP/usr/share/augeas/lenses/dist" CERTBOT_SNAPPED: "True" renew: - command: bin/python3 $SNAP/bin/certbot -q renew + command: bin/python3 -s $SNAP/bin/certbot -q renew daemon: oneshot environment: PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" @@ -40,7 +40,6 @@ parts: certbot: plugin: python source: . - constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt] python-packages: - git+https://github.com/certbot/python-augeas.git@certbot-patched - ./acme @@ -64,7 +63,6 @@ parts: - libpython3.8-stdlib - libpython3.8-minimal - python3-pip - - python3-setuptools - python3-wheel - python3-venv - python3-minimal @@ -74,13 +72,23 @@ parts: # To build cryptography and cffi if needed build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev] build-environment: - - SNAPCRAFT_PYTHON_VENV_ARGS: --system-site-packages - - PIP_NO_BUILD_ISOLATION: "no" + - SNAPCRAFT_PYTHON_VENV_ARGS: --upgrade + # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the + # parts.[part_name].constraints option available in snapcraft.yaml when the Python plugin is + # used. This is done to let these constraints be applied not only on the certbot package + # build, but also on any isolated build that pip could trigger when building wheels for + # dependencies. See https://github.com/certbot/certbot/pull/8443 for more info. + - PIP_CONSTRAINT: $SNAPCRAFT_PART_SRC/snap-constraints.txt + override-build: | + python3 -m venv "${SNAPCRAFT_PART_INSTALL}" + "${SNAPCRAFT_PART_INSTALL}/bin/python3" "${SNAPCRAFT_PART_SRC}/tools/pipstrap.py" + snapcraftctl build override-pull: | - snapcraftctl pull - cd $SNAPCRAFT_PART_SRC - python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt | grep -v python-augeas > snap-constraints.txt - snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" $SNAPCRAFT_PART_SRC/certbot/certbot/__init__.py` + snapcraftctl pull + python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_constraints.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` shared-metadata: plugin: dump source: . diff --git a/tests/letstest/README.md b/tests/letstest/README.md index 4cf6c83c3..76db57153 100644 --- a/tests/letstest/README.md +++ b/tests/letstest/README.md @@ -1,7 +1,6 @@ # letstest Simple AWS testfarm scripts for certbot client testing -- Configures (canned) boulder server - Launches EC2 instances with a given list of AMIs for different distros - Copies certbot repo and puts it on the instances - Runs certbot tests (bash scripts) on all of these @@ -56,11 +55,6 @@ It will take a minute for these instances to shut down and become available agai A folder named `letest-` is also created with a log file from each instance of the test and a file named "results" containing the output above. The tests take quite a while to run. -Also, the way all of the tests work is to check if there is already a boulder server running and if not start one. The boulder server is left running between tests, -and there are known issues if two instances of boulder attempt to be started. After starting your first test, wait until you see "Found existing boulder server:" or if you see output -about creating a boulder server, wait a minute before starting the 2nd test. You only have to do this after starting your first session of tests or after running -the `aws ec2 terminate-instances` command above. - ## Scripts Example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed to them at runtime via environment variables. test_apache2.sh is a useful reference. @@ -73,5 +67,4 @@ See: - https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html Main repos: -- https://github.com/letsencrypt/boulder - https://github.com/letsencrypt/letsencrypt diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml index 8e8e23116..2663782ce 100644 --- a/tests/letstest/apache2_targets.yaml +++ b/tests/letstest/apache2_targets.yaml @@ -1,4 +1,7 @@ # These images are located in us-east-1. +# +# All machines must currently use x86_64 since Pebble does not currently +# publish images for other architectures. targets: #----------------------------------------------------------------------------- @@ -30,12 +33,6 @@ targets: type: ubuntu virt: hvm user: admin - - ami: ami-0dcd54b7d2fff584f - name: debian10_arm64 - type: ubuntu - virt: hvm - user: admin - machine_type: a1.medium - ami: ami-003f19e0e687de1cd name: debian9 type: ubuntu diff --git a/tests/letstest/auto_targets.yaml b/tests/letstest/auto_targets.yaml index 76b3a3dc5..01d410227 100644 --- a/tests/letstest/auto_targets.yaml +++ b/tests/letstest/auto_targets.yaml @@ -31,10 +31,6 @@ targets: virt: hvm user: admin machine_type: a1.medium - # userdata: | - # #cloud-init - # runcmd: - # - [ apt-get, install, -y, curl ] #----------------------------------------------------------------------------- # Other Redhat Distros - ami: ami-0916c408cb02e310b @@ -56,17 +52,6 @@ targets: type: centos virt: hvm user: centos - # centos6 requires EPEL repo added - - ami: ami-1585c46a - name: centos6 - type: centos - virt: hvm - user: centos - userdata: | - #cloud-config - runcmd: - - yum install -y epel-release - - iptables -F - ami: ami-01ca03df4a6012157 name: centos8 type: centos diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index cf9f2899a..1907995c2 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -1,7 +1,6 @@ """ Certbot Integration Test Tool -- Configures (canned) boulder server - Launches EC2 instances with a given list of AMIs for different distros - Copies certbot repo and puts it on the instances - Runs certbot tests (bash scripts) on all of these @@ -28,10 +27,6 @@ see: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html """ - -from __future__ import print_function -from __future__ import with_statement - import argparse import multiprocessing as mp from multiprocessing import Manager @@ -40,11 +35,11 @@ import socket import sys import time import traceback +import urllib.error as urllib_error +import urllib.request as urllib_request import boto3 from botocore.exceptions import ClientError -from six.moves.urllib import error as urllib_error -from six.moves.urllib import request as urllib_request import yaml from fabric import Config @@ -81,12 +76,6 @@ parser.add_argument('--saveinstances', parser.add_argument('--alt_pip', default='', help="server from which to pull candidate release packages") -parser.add_argument('--killboulder', - action='store_true', - help="do not leave a persistent boulder server running") -parser.add_argument('--boulderonly', - action='store_true', - help="only make a boulder server") cl_args = parser.parse_args() # Credential Variables @@ -98,12 +87,11 @@ PROFILE = None if cl_args.aws_profile == 'SET_BY_ENV' else cl_args.aws_profile # Globals #------------------------------------------------------------------------------- -BOULDER_AMI = 'ami-072a9534772bec854' # premade shared boulder AMI 18.04LTS us-east-1 SECURITY_GROUP_NAME = 'certbot-security-group' SENTINEL = None #queue kill signal SUBNET_NAME = 'certbot-subnet' -class Status(object): +class Status: """Possible statuses of client tests.""" PASS = 'pass' FAIL = 'fail' @@ -133,10 +121,6 @@ def make_security_group(vpc): mysg = vpc.create_security_group(GroupName=SECURITY_GROUP_NAME, Description='security group for automated testing') mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22) - mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80) - mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=443, ToPort=443) - # for boulder wfe (http) server - mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=4000, ToPort=4000) # for mosh mysg.authorize_ingress(IpProtocol="udp", CidrIp="0.0.0.0/0", FromPort=60000, ToPort=61000) return mysg @@ -147,22 +131,32 @@ def make_instance(ec2_client, keyname, security_group_id, subnet_id, - machine_type='t2.micro', - userdata=""): #userdata contains bash or cloud-init script + self_destruct, + machine_type='t2.micro'): + """Creates an instance using the given parameters. + + If self_destruct is True, the instance will be configured to shutdown after + 1 hour and to terminate itself on shutdown. + + """ block_device_mappings = _get_block_device_mappings(ec2_client, ami_id) tags = [{'Key': 'Name', 'Value': instance_name}] tag_spec = [{'ResourceType': 'instance', 'Tags': tags}] - return ec2_client.create_instances( - BlockDeviceMappings=block_device_mappings, - ImageId=ami_id, - SecurityGroupIds=[security_group_id], - SubnetId=subnet_id, - KeyName=keyname, - MinCount=1, - MaxCount=1, - UserData=userdata, - InstanceType=machine_type, - TagSpecifications=tag_spec)[0] + kwargs = { + 'BlockDeviceMappings': block_device_mappings, + 'ImageId': ami_id, + 'SecurityGroupIds': [security_group_id], + 'SubnetId': subnet_id, + 'KeyName': keyname, + 'MinCount': 1, + 'MaxCount': 1, + 'InstanceType': machine_type, + 'TagSpecifications': tag_spec + } + if self_destruct: + kwargs['InstanceInitiatedShutdownBehavior'] = 'terminate' + kwargs['UserData'] = '#!/bin/bash\nshutdown -P +60\n' + return ec2_client.create_instances(**kwargs)[0] def _get_block_device_mappings(ec2_client, ami_id): """Returns the list of block device mappings to ensure cleanup. @@ -183,23 +177,6 @@ def _get_block_device_mappings(ec2_client, ami_id): # Helper Routines #------------------------------------------------------------------------------- -def block_until_http_ready(urlstring, wait_time=10, timeout=240): - "Blocks until server at urlstring can respond to http requests" - server_ready = False - t_elapsed = 0 - while not server_ready and t_elapsed < timeout: - try: - sys.stdout.write('.') - sys.stdout.flush() - req = urllib_request.Request(urlstring) - response = urllib_request.urlopen(req) - #if response.code == 200: - server_ready = True - except urllib_error.URLError: - pass - time.sleep(wait_time) - t_elapsed += wait_time - def block_until_ssh_open(ipstring, wait_time=10, timeout=120): "Blocks until server at ipstring has an open port 22" reached = False @@ -278,26 +255,15 @@ def deploy_script(cxn, scriptpath, *args): args_str = ' '.join(args) cxn.run('./'+scriptfile+' '+args_str) -def run_boulder(cxn): - boulder_path = '$GOPATH/src/github.com/letsencrypt/boulder' - cxn.run('cd %s && sudo docker-compose up -d' % boulder_path) - -def config_and_launch_boulder(cxn, instance): - # yes, we're hardcoding the gopath. it's a predetermined AMI. - with cxn.prefix('export GOPATH=/home/ubuntu/gopath'): - deploy_script(cxn, 'scripts/boulder_config.sh') - run_boulder(cxn) - -def install_and_launch_certbot(cxn, instance, boulder_url, target, log_dir): +def install_and_launch_certbot(cxn, instance, target, log_dir): local_repo_to_remote(cxn, log_dir) # This needs to be like this, I promise. 1) The env argument to run doesn't work. # See https://github.com/fabric/fabric/issues/1744. 2) prefix() sticks an && between # the commands, so it needs to be exports rather than no &&s in between for the script subshell. - with cxn.prefix('export BOULDER_URL=%s && export PUBLIC_IP=%s && export PRIVATE_IP=%s && ' + with cxn.prefix('export PUBLIC_IP=%s && export PRIVATE_IP=%s && ' 'export PUBLIC_HOSTNAME=%s && export PIP_EXTRA_INDEX_URL=%s && ' 'export OS_TYPE=%s' % - (boulder_url, - instance.public_ip_address, + (instance.public_ip_address, instance.private_ip_address, instance.public_dns_name, cl_args.alt_pip, @@ -313,7 +279,7 @@ def grab_certbot_log(cxn): 'cat ./certbot.log; else echo "[nolocallog]"; fi\'') -def create_client_instance(ec2_client, target, security_group_id, subnet_id): +def create_client_instance(ec2_client, target, security_group_id, subnet_id, self_destruct): """Create a single client instance for running tests.""" if 'machine_type' in target: machine_type = target['machine_type'] @@ -322,10 +288,6 @@ def create_client_instance(ec2_client, target, security_group_id, subnet_id): else: # 32 bit systems machine_type = 'c1.medium' - if 'userdata' in target: - userdata = target['userdata'] - else: - userdata = '' name = 'le-%s'%target['name'] print(name, end=" ") return make_instance(ec2_client, @@ -335,10 +297,10 @@ def create_client_instance(ec2_client, target, security_group_id, subnet_id): machine_type=machine_type, security_group_id=security_group_id, subnet_id=subnet_id, - userdata=userdata) + self_destruct=self_destruct) -def test_client_process(fab_config, inqueue, outqueue, boulder_url, log_dir): +def test_client_process(fab_config, inqueue, outqueue, log_dir): cur_proc = mp.current_process() for inreq in iter(inqueue.get, SENTINEL): ii, instance_id, target = inreq @@ -360,7 +322,7 @@ def test_client_process(fab_config, inqueue, outqueue, boulder_url, log_dir): with Connection(host_string, config=fab_config) as cxn: try: - install_and_launch_certbot(cxn, instance, boulder_url, target, log_dir) + install_and_launch_certbot(cxn, instance, target, log_dir) outqueue.put((ii, target, Status.PASS)) print("%s - %s SUCCESS"%(target['ami'], target['name'])) except: @@ -379,15 +341,13 @@ def test_client_process(fab_config, inqueue, outqueue, boulder_url, log_dir): pass -def cleanup(cl_args, instances, targetlist, boulder_server, log_dir): +def cleanup(cl_args, instances, targetlist, log_dir): print('Logs in ', log_dir) # If lengths of instances and targetlist aren't equal, instances failed to # start before running tests so leaving instances running for debugging # isn't very useful. Let's cleanup after ourselves instead. if len(instances) != len(targetlist) or not cl_args.saveinstances: print('Terminating EC2 Instances') - if cl_args.killboulder: - boulder_server.terminate() for instance in instances: instance.terminate() else: @@ -477,63 +437,18 @@ def main(): security_group_id = make_security_group(vpc).id time.sleep(30) - boulder_preexists = False - boulder_servers = ec2_client.instances.filter(Filters=[ - {'Name': 'tag:Name', 'Values': ['le-boulderserver']}, - {'Name': 'instance-state-name', 'Values': ['running']}]) - - boulder_server = next(iter(boulder_servers), None) - - print("Requesting Instances...") - if boulder_server: - print("Found existing boulder server:", boulder_server) - boulder_preexists = True - else: - print("Can't find a boulder server, starting one...") - boulder_server = make_instance(ec2_client, - 'le-boulderserver', - BOULDER_AMI, - KEYNAME, - machine_type='t2.micro', - #machine_type='t2.medium', - security_group_id=security_group_id, - subnet_id=subnet_id) - instances = [] try: - if not cl_args.boulderonly: - print("Creating instances: ", end="") - for target in targetlist: - instances.append( - create_client_instance(ec2_client, target, - security_group_id, subnet_id) - ) - print() - - # Configure and launch boulder server - #------------------------------------------------------------------------------- - print("Waiting on Boulder Server") - boulder_server = block_until_instance_ready(boulder_server) - print(" server %s"%boulder_server) - - - # host_string defines the ssh user and host for connection - host_string = "ubuntu@%s"%boulder_server.public_ip_address - print("Boulder Server at (SSH):", host_string) - if not boulder_preexists: - print("Configuring and Launching Boulder") - with Connection(host_string, config=fab_config) as boulder_cxn: - config_and_launch_boulder(boulder_cxn, boulder_server) - # blocking often unnecessary, but cheap EC2 VMs can get very slow - block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, - wait_time=10, timeout=500) - - boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address - print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) - print("Boulder Server at (EC2 private ip): %s"%boulder_url) - - if cl_args.boulderonly: - sys.exit(0) + print("Creating instances: ", end="") + # If we want to preserve instances, do not have them self-destruct. + self_destruct = not cl_args.saveinstances + for target in targetlist: + instances.append( + create_client_instance(ec2_client, target, + security_group_id, subnet_id, + self_destruct) + ) + print() # Install and launch client scripts in parallel #------------------------------------------------------------------------------- @@ -551,7 +466,7 @@ def main(): # initiate process execution - client_process_args=(fab_config, inqueue, outqueue, boulder_url, log_dir) + client_process_args=(fab_config, inqueue, outqueue, log_dir) for i in range(num_processes): p = mp.Process(target=test_client_process, args=client_process_args) jobs.append(p) @@ -602,7 +517,7 @@ def main(): sys.exit(1) finally: - cleanup(cl_args, instances, targetlist, boulder_server, log_dir) + cleanup(cl_args, instances, targetlist, log_dir) if __name__ == '__main__': diff --git a/tests/letstest/scripts/bootstrap_os_packages.sh b/tests/letstest/scripts/bootstrap_os_packages.sh index 96506282b..7ad93f63e 100755 --- a/tests/letstest/scripts/bootstrap_os_packages.sh +++ b/tests/letstest/scripts/bootstrap_os_packages.sh @@ -98,41 +98,6 @@ BootstrapRpmCommonBase() { fi } -# This bootstrap concerns old RedHat-based distributions that do not ship by default -# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing -# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6. -BootstrapRpmPython3Legacy() { - # Tested with: - # - CentOS 6 - - InitializeRPMCommonBase - - if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then - echo "To use Certbot on this operating system, packages from the SCL repository need to be installed." - if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then - error "Enable the SCL repository and try running Certbot again." - exit 1 - fi - if ! "${TOOL}" install -y centos-release-scl; then - error "Could not enable SCL. Aborting bootstrap!" - exit 1 - fi - fi - - # CentOS 6 must use rh-python36 from SCL - if "${TOOL}" list rh-python36 >/dev/null 2>&1; then - python_pkgs="rh-python36-python - rh-python36-python-virtualenv - rh-python36-python-devel - " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 - fi - - BootstrapRpmCommonBase "${python_pkgs}" -} - BootstrapRpmPython3() { InitializeRPMCommonBase @@ -154,16 +119,9 @@ if [ -f /etc/debian_version ]; then } elif [ -f /etc/redhat-release ]; then DeterminePythonVersion - # Handle legacy RPM distributions - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapRpmPython3Legacy - } - else - Bootstrap() { - BootstrapRpmPython3 - } - fi + Bootstrap() { + BootstrapRpmPython3 + } fi diff --git a/tests/letstest/scripts/boulder_config.sh b/tests/letstest/scripts/boulder_config.sh deleted file mode 100755 index b99bbabbe..000000000 --- a/tests/letstest/scripts/boulder_config.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -x - -# Configures and Launches Boulder Server installed on -# us-east-1 ami-072a9534772bec854 bouldertestserver3 (boulder commit b24fe7c3ea4) - -# fetch instance data from EC2 metadata service -public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) -public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) -private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) - -# set to public DNS resolver -resolver_ip=8.8.8.8 -resolver=$resolver_ip':53' - -# modifies integration testing boulder setup for local AWS VPC network -# connections instead of localhost -cd $GOPATH/src/github.com/letsencrypt/boulder -# change test ports to real -sed -i '/httpPort/ s/5002/80/' ./test/config/va.json -sed -i '/httpsPort/ s/5001/443/' ./test/config/va.json -sed -i '/tlsPort/ s/5001/443/' ./test/config/va.json -# set dns resolver -sed -i 's/"127.0.0.1:8053",/"'$resolver'"/' ./test/config/va.json -sed -i 's/"127.0.0.1:8054"//' ./test/config/va.json diff --git a/tests/letstest/scripts/boulder_install.sh b/tests/letstest/scripts/boulder_install.sh deleted file mode 100755 index 5161de374..000000000 --- a/tests/letstest/scripts/boulder_install.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -x - -# Check out special branch until latest docker changes land in Boulder master. -git clone -b docker-integration https://github.com/letsencrypt/boulder $BOULDERPATH -cd $BOULDERPATH -FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') -sed -i "s/FAKE_DNS: .*/FAKE_DNS: $FAKE_DNS/" docker-compose.yml -docker-compose up -d diff --git a/tests/letstest/scripts/set_python_envvars.sh b/tests/letstest/scripts/set_python_envvars.sh deleted file mode 100755 index 668444209..000000000 --- a/tests/letstest/scripts/set_python_envvars.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -# This is a simple script that can be sourced to set Python environment -# variables for use in Certbot's letstest test farm tests. - -# Some distros like Fedora may only have an executable named python3 installed. -if command -v python; then - PYTHON_NAME="python" - VENV_SCRIPT="tools/venv.py" - VENV_PATH="venv" -else - # We could check for "python2" here, however, the addition of "python3" - # only systems is what necessitated this change so checking for "python2" - # isn't necessary. - PYTHON_NAME="python3" - VENV_PATH="venv3" - VENV_SCRIPT="tools/venv3.py" -fi diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index ba3d94379..77dc35f1e 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -7,7 +7,7 @@ if [ "$OS_TYPE" = "ubuntu" ] then CONFFILE=/etc/apache2/sites-available/000-default.conf sudo apt-get update - sudo apt-get -y --no-upgrade install apache2 #curl + sudo apt-get -y --no-upgrade install apache2 curl sudo apt-get -y install realpath # needed for test-apache-conf # For apache 2.4, set up ServerName sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE @@ -64,17 +64,41 @@ if [ $? -ne 0 ] ; then exit 1 fi -if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then - # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment - # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap - PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" +tools/venv.py -e acme[dev] -e certbot[dev,docs] -e certbot-apache -e certbot-ci +PEBBLE_LOGS="acme_server.log" +PEBBLE_URL="https://localhost:14000/dir" +# We configure Pebble to use port 80 for http-01 validation rather than an +# alternate port because: +# 1) It allows us to test with Apache configurations that are more realistic +# and closer to the default configuration on various OSes. +# 2) As of writing this, Certbot's Apache plugin requires there to be an +# existing virtual host for the port used for http-01 validation. +venv/bin/run_acme_server --http-01-port 80 > "${PEBBLE_LOGS}" 2>&1 & + +DumpPebbleLogs() { + if [ -f "${PEBBLE_LOGS}" ] ; then + echo "Pebble's logs were:" + cat "${PEBBLE_LOGS}" + fi +} + +for n in $(seq 1 150) ; do + if curl --insecure "${PEBBLE_URL}" 2>/dev/null; then + break + else + echo "waiting for pebble" + sleep 1 + fi +done +if ! curl --insecure "${PEBBLE_URL}" 2>/dev/null; then + echo "timed out waiting for pebble to start" + DumpPebbleLogs + exit 1 fi -tools/venv3.py -e acme[dev] -e certbot[dev,docs] -e certbot-apache - -sudo "venv3/bin/certbot" -v --debug --text --agree-tos \ +sudo "venv/bin/certbot" -v --debug --text --agree-tos --no-verify-ssl \ --renew-by-default --redirect --register-unsafely-without-email \ - --domain $PUBLIC_HOSTNAME --server $BOULDER_URL + --domain "${PUBLIC_HOSTNAME}" --server "${PEBBLE_URL}" if [ $? -ne 0 ] ; then FAIL=1 fi @@ -89,15 +113,15 @@ elif [ "$OS_TYPE" = "centos" ]; then fi OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9) APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8) -"venv3/bin/python" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" +"venv/bin/python" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" if [ $? -ne 0 ] ; then FAIL=1 fi if [ "$OS_TYPE" = "ubuntu" ] ; then - export SERVER="$BOULDER_URL" - "venv3/bin/tox" -e apacheconftest + export SERVER="${PEBBLE_URL}" + "venv/bin/tox" -e apacheconftest else echo Not running hackish apache tests on $OS_TYPE fi @@ -108,5 +132,6 @@ fi # return error if any of the subtests failed if [ "$FAIL" = 1 ] ; then + DumpPebbleLogs exit 1 fi diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 51ff640c5..3964364e1 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -1,7 +1,7 @@ #!/bin/bash -xe set -o pipefail -# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL +# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME # are dynamically set at execution cd letsencrypt @@ -33,19 +33,22 @@ if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 | exit 1 fi -# This script sets the environment variables PYTHON_NAME, VENV_PATH, and -# VENV_SCRIPT based on the version of Python available on the system. For -# instance, Fedora uses Python 3 and Python 2 is not installed. -. tests/letstest/scripts/set_python_envvars.sh +if command -v python; then + PYTHON_NAME="python" +else + PYTHON_NAME="python3" +fi # Now that python and openssl have been installed, we can set up a fake server # to provide a new version of letsencrypt-auto. First, we start the server and # directory to be served. MY_TEMP_DIR=$(mktemp -d) PORT_FILE="$MY_TEMP_DIR/port" +LOG_FILE="$MY_TEMP_DIR/log" SERVER_PATH=$("$PYTHON_NAME" tools/readlink.py tools/simple_http_server.py) cd "$MY_TEMP_DIR" -"$PYTHON_NAME" "$SERVER_PATH" 0 > $PORT_FILE & +# We set PYTHONUNBUFFERED to disable buffering of output to LOG_FILE +PYTHONUNBUFFERED=1 "$PYTHON_NAME" "$SERVER_PATH" 0 > $PORT_FILE 2> "$LOG_FILE" & SERVER_PID=$! trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT cd ~- @@ -105,15 +108,10 @@ if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python exit 1 fi -# On systems like Debian where certbot-auto is deprecated, we expect it to -# leave existing Certbot installations unmodified so we check for the same -# version that was initially installed below. Once certbot-auto is deprecated -# on RHEL systems, we can unconditionally check for INITIAL_VERSION. -if [ -f /etc/debian_version ]; then - EXPECTED_VERSION="$INITIAL_VERSION" -else - EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) -fi +# Since certbot-auto is deprecated, we expect it to leave existing Certbot +# installations unmodified so we check for the same version that was initially +# installed below. +EXPECTED_VERSION="$INITIAL_VERSION" if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then echo unexpected certbot version found @@ -125,21 +123,38 @@ if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then exit 1 fi -if [ "$RUN_RHEL6_TESTS" = 1 ]; then - # Add the SCL python release to PATH in order to resolve python3 command - PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" - if ! command -v python3; then - echo "Python3 wasn't properly installed" - exit 1 - fi - if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1)" != 3 ]; then - echo "Python3 wasn't used in venv!" - exit 1 - fi +# Now let's test if letsencrypt-auto still tries to upgrade to a new version. +# Regardless of the OS, versions of the script with development version numbers +# ending in .dev0 will not upgrade. See +# https://github.com/certbot/certbot/blob/bdfb9f19c4086a60ef010d2431768850c26d838a/certbot-auto#L1947-L1948. +# In order to test the process of different OSes setting NO_SELF_UPGRADE as +# part of the script's deprecation, we make use of the fact that +# letsencrypt-auto should still attempt to fetch the version number from PyPI +# even if it has a development version number unless NO_SELF_UPGRADE is set in +# which case all of that logic should be skipped. +# +# First we make a copy of the current server logs. +PREVIOUS_LOG_FILE="$MY_TEMP_DIR/previous-log" +cp "$LOG_FILE" "$PREVIOUS_LOG_FILE" - if [ "$("$PYTHON_NAME" tools/readlink.py $OLD_VENV_PATH)" != "/opt/eff.org/certbot/venv" ]; then - echo symlink from old venv path not properly created! - exit 1 - fi +# Next we run letsencrypt-auto and make sure there were no problems checking +# for updates, the Certbot install still works, the version number is what +# we expect, and it prints a message about not receiving updates. +if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then + echo "Had problems checking for updates!" + exit 1 +fi +if ! ./letsencrypt-auto -v --debug --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then + echo unexpected certbot version found + exit 1 +fi +if ! ./letsencrypt-auto -v --debug --version 2>&1 | grep "will no longer receive updates" ; then + echo script did not print warning about not receiving updates! + exit 1 +fi + +# Finally, we check if our local server received more requests. +if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then + echo our local server received unexpected requests + exit 1 fi -echo upgrade appeared to be successful diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh index 15cf9ee1b..9573ab690 100755 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh @@ -1,7 +1,7 @@ #!/bin/bash -x set -eo pipefail -# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution +# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME are dynamically set at execution # with curl, instance metadata available from EC2 metadata service: #public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) @@ -16,58 +16,14 @@ sudo chown root "$LE_AUTO_PATH" sudo chmod 0755 "$LE_AUTO_PATH" export PATH="$LE_AUTO_DIR:$PATH" -# On systems like Debian where certbot-auto is deprecated, we expect -# certbot-auto to error and refuse to install Certbot. Once certbot-auto is -# deprecated on RHEL systems, we can unconditionally run this code. -if [ -f /etc/debian_version ]; then - set +o pipefail - if ! letsencrypt-auto --debug --version | grep "Certbot cannot be installed."; then - echo "letsencrypt-auto didn't report being uninstallable." - exit 1 - fi - if [ ${PIPESTATUS[0]} != 1 ]; then - echo "letsencrypt-auto didn't exit with status 1 as expected" - exit 1 - fi - # letsencrypt-auto is deprecated and cannot be installed on this system so - # we cannot run the rest of this test. - exit 0 -fi - -letsencrypt-auto --os-packages-only --debug --version - -# This script sets the environment variables PYTHON_NAME, VENV_PATH, and -# VENV_SCRIPT based on the version of Python available on the system. For -# instance, Fedora uses Python 3 and Python 2 is not installed. -. tests/letstest/scripts/set_python_envvars.sh - -# Create a venv-like layout at the old virtual environment path to test that a -# symlink is properly created when letsencrypt-auto runs. -HOME=${HOME:-~root} -XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} -OLD_VENV_BIN="$XDG_DATA_HOME/letsencrypt/bin" -mkdir -p "$OLD_VENV_BIN" -touch "$OLD_VENV_BIN/letsencrypt" - -letsencrypt-auto certonly --no-self-upgrade -v --standalone --debug \ - --text --agree-tos \ - --renew-by-default --redirect \ - --register-unsafely-without-email \ - --domain $PUBLIC_HOSTNAME --server $BOULDER_URL - -LINK_PATH=$("$PYTHON_NAME" tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt) -if [ "$LINK_PATH" != "/opt/eff.org/certbot/venv" ]; then - echo symlink from old venv path not properly created! +# Since certbot-auto is deprecated, we expect certbot-auto to error and +# refuse to install Certbot. +set +o pipefail +if ! letsencrypt-auto --debug --version | grep "Certbot cannot be installed."; then + echo "letsencrypt-auto didn't report being uninstallable." exit 1 fi - -if ! letsencrypt-auto --help --no-self-upgrade | grep -F "letsencrypt-auto [SUBCOMMAND]"; then - echo "letsencrypt-auto not included in help output!" - exit 1 -fi - -OUTPUT_LEN=$(letsencrypt-auto --install-only --no-self-upgrade --quiet 2>&1 | wc -c) -if [ "$OUTPUT_LEN" != 0 ]; then - echo letsencrypt-auto produced unexpected output! +if [ ${PIPESTATUS[0]} != 1 ]; then + echo "letsencrypt-auto didn't exit with status 1 as expected" exit 1 fi diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index e3d9a8b80..becdd6d9a 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -3,28 +3,30 @@ cd letsencrypt BOOTSTRAP_SCRIPT="tests/letstest/scripts/bootstrap_os_packages.sh" -VENV_PATH=venv3 +VENV_PATH=venv # install OS packages sudo $BOOTSTRAP_SCRIPT -if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then - # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment - # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap - PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" -fi - # setup venv # We strip the hashes because the venv creation script includes unhashed # constraints in the commands given to pip and the mix of hashed and unhashed # packages makes pip error out. -python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > requirements.txt -# We also strip out the requirement for enum34 because it cannot be installed -# in newer versions of Python 3, tools/strip_hashes.py removes the environment -# marker that'd normally prevent it from being installed, and this package is -# not needed for any OS tested here. -sed -i '/enum34/d' requirements.txt -CERTBOT_PIP_NO_BINARY=:all: tools/venv3.py --requirement requirements.txt +python3 tools/strip_hashes.py tools/pipstrap_constraints.txt > constraints.txt +python3 tools/strip_hashes.py tools/certbot_constraints.txt > requirements.txt + +# We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7 +# because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been +# dropped on cryptography>=3.2 and pyOpenSSL>=20.0.0. +# Using this old version of OpenSSL would break the cryptography and pyOpenSSL wheels builds. +if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then + sed -i 's|cryptography==.*|cryptography==3.1.1|g' requirements.txt + sed -i 's|pyOpenSSL==.*|pyOpenSSL==19.1.0|g' requirements.txt +fi + +python3 -m venv $VENV_PATH +$VENV_PATH/bin/python3 tools/pipstrap.py +PIP_CONSTRAINT=constraints.txt PIP_NO_BINARY=:all: $VENV_PATH/bin/python3 -m pip install --requirement requirements.txt . "$VENV_PATH/bin/activate" # pytest is needed to run tests on some of our packages so we install a pinned version here. tools/pip_install.py pytest diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index f62584709..858fc1f18 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -9,18 +9,12 @@ LE_AUTO="$REPO_ROOT/letsencrypt-auto-source/letsencrypt-auto" LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" MODULES="acme certbot certbot-apache certbot-nginx" PIP_INSTALL="tools/pip_install.py" -VENV_NAME=venv3 +VENV_NAME=venv BOOTSTRAP_SCRIPT="$REPO_ROOT/tests/letstest/scripts/bootstrap_os_packages.sh" -VENV_SCRIPT="tools/venv3.py" +VENV_SCRIPT="tools/venv.py" sudo $BOOTSTRAP_SCRIPT -if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then - # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment - # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap - PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" -fi - cd $REPO_ROOT $VENV_SCRIPT . $VENV_NAME/bin/activate diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 29edd1552..97c775f6c 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -52,17 +52,6 @@ targets: type: centos virt: hvm user: centos - # centos6 requires EPEL repo added - - ami: ami-1585c46a - name: centos6 - type: centos - virt: hvm - user: centos - userdata: | - #cloud-config - runcmd: - - yum install -y epel-release - - iptables -F - ami: ami-01ca03df4a6012157 name: centos8 type: centos diff --git a/tests/lock_test.py b/tests/lock_test.py index 56399c874..f310b5753 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -1,6 +1,4 @@ """Tests to ensure the lock order is preserved.""" -from __future__ import print_function - import atexit import datetime import functools diff --git a/tests/modification-check.py b/tests/modification-check.py index 7a69fb1db..357b25350 100755 --- a/tests/modification-check.py +++ b/tests/modification-check.py @@ -7,16 +7,13 @@ import shutil import subprocess import sys import tempfile +from urllib.request import urlretrieve -try: - from urllib.request import urlretrieve -except ImportError: - from urllib import urlretrieve def find_repo_path(): return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -# We do not use filecmp.cmp to take advantage of universal newlines +# We do not use filecmp.cmp to take advantage of universal newlines # handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. # As a consequence, this function will not work correctly if executed by Python 2.x on Windows. # But it will work correctly on Linux for any version, because every file tested will be LF. @@ -50,10 +47,10 @@ def validate_scripts_content(repo_path, temp_cwd): errors = True else: shutil.copyfile( - os.path.join(repo_path, 'certbot-auto'), + os.path.join(repo_path, 'certbot-auto'), os.path.join(temp_cwd, 'local-auto')) shutil.copy(os.path.normpath(os.path.join( - repo_path, + repo_path, 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) # Compare file against current version in the target branch @@ -72,7 +69,7 @@ def validate_scripts_content(repo_path, temp_cwd): latest_version = subprocess.check_output( [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) subprocess.check_call( - [sys.executable, 'fetch.py', '--le-auto-script', + [sys.executable, 'fetch.py', '--le-auto-script', 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) if compare_files( os.path.join(temp_cwd, 'letsencrypt-auto'), diff --git a/tools/_release.sh b/tools/_release.sh index e17332f30..e9e0e49f0 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -216,8 +216,13 @@ fi # ensure we have the latest built version of leauto letsencrypt-auto-source/build.py -# and that it's signed correctly -tools/offline-sigrequest.sh || true +# Now we have to sign the built version of leauto. +SignLEAuto() { + yubico-piv-tool -a verify-pin --sign -s 9c -i letsencrypt-auto-source/letsencrypt-auto -o letsencrypt-auto-source/letsencrypt-auto.sig +} + +# Loop until letsencrypt-auto is signed correctly. +SignLEAuto || true while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ letsencrypt-auto-source/letsencrypt-auto.sig \ letsencrypt-auto-source/letsencrypt-auto ; do @@ -225,7 +230,7 @@ while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ read -p "Would you like this script to try and sign it again [Y/n]?" response case $response in [yY][eE][sS]|[yY]|"") - tools/offline-sigrequest.sh || true;; + SignLEAuto || true;; *) ;; esac diff --git a/tools/_venv_common.py b/tools/_venv_common.py deleted file mode 100644 index 58c05ed09..000000000 --- a/tools/_venv_common.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -"""Aids in creating a developer virtual environment for Certbot. - -When this module is run as a script, it takes the arguments that should -be passed to pip to install the Certbot packages as command line -arguments. The virtual environment will be created with the name "venv" -in the current working directory and will use the default version of -Python for the virtualenv executable in your PATH. You can change the -name of the virtual environment by setting the environment variable -VENV_NAME. -""" - -from __future__ import print_function - -from distutils.version import LooseVersion -import glob -import os -import re -import shutil -import subprocess -import sys -import time - -REQUIREMENTS = [ - '-e acme[dev]', - '-e certbot[dev,docs]', - '-e certbot-apache', - '-e certbot-dns-cloudflare', - '-e certbot-dns-cloudxns', - '-e certbot-dns-digitalocean', - '-e certbot-dns-dnsimple', - '-e certbot-dns-dnsmadeeasy', - '-e certbot-dns-gehirn', - '-e certbot-dns-google', - '-e certbot-dns-linode', - '-e certbot-dns-luadns', - '-e certbot-dns-nsone', - '-e certbot-dns-ovh', - '-e certbot-dns-rfc2136', - '-e certbot-dns-route53', - '-e certbot-dns-sakuracloud', - '-e certbot-nginx', - '-e certbot-compatibility-test', - '-e certbot-ci', -] - -VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') - - -class PythonExecutableNotFoundError(Exception): - pass - - -def find_python_executable(python_major): - # type: (int) -> str - """ - Find the relevant python executable that is of the given python major version. - Will test, in decreasing priority order: - - * the current Python interpreter - * 'pythonX' executable in PATH (with X the given major version) if available - * 'python' executable in PATH if available - * Windows Python launcher 'py' executable in PATH if available - - Incompatible python versions for Certbot will be evicted (e.g. Python 3 - versions less than 3.6). - - :param int python_major: the Python major version to target (2 or 3) - :rtype: str - :return: the relevant python executable path - :raise RuntimeError: if no relevant python executable path could be found - """ - python_executable_path = None - - # First try, current python executable - if _check_version('{0}.{1}.{2}'.format( - sys.version_info[0], sys.version_info[1], sys.version_info[2]), python_major): - return sys.executable - - # Second try, with python executables in path - versions_to_test = ['2.7', '2', ''] if python_major == 2 else ['3', ''] - for one_version in versions_to_test: - try: - one_python = 'python{0}'.format(one_version) - output = subprocess.check_output([one_python, '--version'], - universal_newlines=True, stderr=subprocess.STDOUT) - if _check_version(output.strip().split()[1], python_major): - return subprocess.check_output([one_python, '-c', - 'import sys; sys.stdout.write(sys.executable);'], - universal_newlines=True) - except (subprocess.CalledProcessError, OSError): - pass - - # Last try, with Windows Python launcher - try: - env_arg = '-{0}'.format(python_major) - output_version = subprocess.check_output(['py', env_arg, '--version'], - universal_newlines=True, stderr=subprocess.STDOUT) - if _check_version(output_version.strip().split()[1], python_major): - return subprocess.check_output(['py', env_arg, '-c', - 'import sys; sys.stdout.write(sys.executable);'], - universal_newlines=True) - except (subprocess.CalledProcessError, OSError): - pass - - if not python_executable_path: - raise RuntimeError('Error, no compatible Python {0} executable for Certbot could be found.' - .format(python_major)) - - -def _check_version(version_str, major_version): - search = VERSION_PATTERN.search(version_str) - - if not search: - return False - - version = (int(search.group(1)), int(search.group(2))) - - minimal_version_supported = (2, 7) - if major_version == 3: - minimal_version_supported = (3, 6) - - if version >= minimal_version_supported: - return True - - print('Incompatible python version for Certbot found: {0}'.format(version_str)) - return False - - -def subprocess_with_print(cmd, env=None, shell=False): - if env is None: - env = os.environ - print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) - subprocess.check_call(cmd, env=env, shell=shell) - - -def subprocess_output_with_print(cmd, env=None, shell=False): - if env is None: - env = os.environ - print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) - return subprocess.check_output(cmd, env=env, shell=shell) - - -def get_venv_python_path(venv_path): - python_linux = os.path.join(venv_path, 'bin/python') - if os.path.isfile(python_linux): - return os.path.abspath(python_linux) - python_windows = os.path.join(venv_path, 'Scripts\\python.exe') - if os.path.isfile(python_windows): - return os.path.abspath(python_windows) - - raise ValueError(( - 'Error, could not find python executable in venv path {0}: is it a valid venv ?' - .format(venv_path))) - - -def prepare_venv_path(venv_name): - """Determines the venv path and prepares it for use. - - This function cleans up any Python eggs in the current working directory - and ensures the venv path is available for use. The path used is the - VENV_NAME environment variable if it is set and venv_name otherwise. If - there is already a directory at the desired path, the existing directory is - renamed by appending a timestamp to the directory name. - - :param str venv_name: The name or path at where the virtual - environment should be created if VENV_NAME isn't set. - - :returns: path where the virtual environment should be created - :rtype: str - - """ - for path in glob.glob('*.egg-info'): - if os.path.isdir(path): - shutil.rmtree(path) - else: - os.remove(path) - - env_venv_name = os.environ.get('VENV_NAME') - if env_venv_name: - print('Creating venv at {0}' - ' as specified in VENV_NAME'.format(env_venv_name)) - venv_name = env_venv_name - - if os.path.isdir(venv_name): - os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - - return venv_name - - -def install_packages(venv_name, pip_args): - """Installs packages in the given venv. - - :param str venv_name: The name or path at where the virtual - environment should be created. - :param pip_args: Command line arguments that should be given to - pip to install packages - :type pip_args: `list` of `str` - - """ - # Using the python executable from venv, we ensure to execute following commands in this venv. - py_venv = get_venv_python_path(venv_name) - subprocess_with_print([py_venv, os.path.abspath('tools/pipstrap.py')]) - # We only use this value during pip install because: - # 1) We're really only adding it for installing cryptography, which happens here, and - # 2) There are issues with calling it along with VIRTUALENV_NO_DOWNLOAD, which applies at the - # steps above, not during pip install. - env_pip_no_binary = os.environ.get('CERTBOT_PIP_NO_BINARY') - if env_pip_no_binary: - # Check OpenSSL version. If it's too low, don't apply the env variable. - openssl_version_string = str(subprocess_output_with_print(['openssl', 'version'])) - matches = re.findall(r'OpenSSL ([^ ]+) ', openssl_version_string) - if not matches: - print('Could not find OpenSSL version, not setting PIP_NO_BINARY.') - else: - openssl_version = matches[0] - - if LooseVersion(openssl_version) >= LooseVersion('1.0.2'): - print('Setting PIP_NO_BINARY to {0}' - ' as specified in CERTBOT_PIP_NO_BINARY'.format(env_pip_no_binary)) - os.environ['PIP_NO_BINARY'] = env_pip_no_binary - else: - print('Not setting PIP_NO_BINARY, as OpenSSL version is too old.') - command = [py_venv, os.path.abspath('tools/pip_install.py')] - command.extend(pip_args) - subprocess_with_print(command) - if 'PIP_NO_BINARY' in os.environ: - del os.environ['PIP_NO_BINARY'] - - if os.path.isdir(os.path.join(venv_name, 'bin')): - # Linux/OSX specific - print('-------------------------------------------------------------------') - print('Please run the following command to activate developer environment:') - print('source {0}/bin/activate'.format(venv_name)) - print('-------------------------------------------------------------------') - elif os.path.isdir(os.path.join(venv_name, 'Scripts')): - # Windows specific - print('---------------------------------------------------------------------------') - print('Please run one of the following commands to activate developer environment:') - print('{0}\\Scripts\\activate.bat (for Batch)'.format(venv_name)) - print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name)) - print('---------------------------------------------------------------------------') - else: - raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) diff --git a/tools/certbot_constraints.txt b/tools/certbot_constraints.txt new file mode 100644 index 000000000..77bfef9db --- /dev/null +++ b/tools/certbot_constraints.txt @@ -0,0 +1,262 @@ +# This is the flattened list of pinned packages to build certbot deployable artifacts. +# To generate this, do (with docker and package hashin installed): +# ``` +# tools/rebuild_certbot_contraints.py \ +# tools/certbot_constraints.txt +# ``` +# If you want to update a single dependency, run commands similar to these: +# ``` +# pip install hashin +# hashin -r dependency-requirements.txt cryptography==1.5.2 +# ``` +ConfigArgParse==1.2.3 \ + --hash=sha256:edd17be986d5c1ba2e307150b8e5f5107aba125f3574dddd02c85d5cdcfd37dc +certifi==2020.12.5 \ + --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ + --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 +cffi==1.14.4 \ + --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \ + --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \ + --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \ + --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \ + --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \ + --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \ + --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \ + --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \ + --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \ + --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \ + --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \ + --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \ + --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \ + --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \ + --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \ + --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \ + --hash=sha256:7ef7d4ced6b325e92eb4d3502946c78c5367bc416398d387b39591532536734e \ + --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \ + --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \ + --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \ + --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \ + --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \ + --hash=sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01 \ + --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \ + --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \ + --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \ + --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \ + --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \ + --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \ + --hash=sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e \ + --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \ + --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \ + --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \ + --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \ + --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \ + --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \ + --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f +chardet==4.0.0 \ + --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ + --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 +configobj==5.0.6 \ + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 +cryptography==3.3.2 \ + --hash=sha256:0d7b69674b738068fa6ffade5c962ecd14969690585aaca0a1b1fc9058938a72 \ + --hash=sha256:1bd0ccb0a1ed775cd7e2144fe46df9dc03eefd722bbcf587b3e0616ea4a81eff \ + --hash=sha256:3c284fc1e504e88e51c428db9c9274f2da9f73fdf5d7e13a36b8ecb039af6e6c \ + --hash=sha256:49570438e60f19243e7e0d504527dd5fe9b4b967b5a1ff21cc12b57602dd85d3 \ + --hash=sha256:541dd758ad49b45920dda3b5b48c968f8b2533d8981bcdb43002798d8f7a89ed \ + --hash=sha256:5a60d3780149e13b7a6ff7ad6526b38846354d11a15e21068e57073e29e19bed \ + --hash=sha256:7951a966613c4211b6612b0352f5bf29989955ee592c4a885d8c7d0f830d0433 \ + --hash=sha256:922f9602d67c15ade470c11d616f2b2364950602e370c76f0c94c94ae672742e \ + --hash=sha256:a0f0b96c572fc9f25c3f4ddbf4688b9b38c69836713fb255f4a2715d93cbaf44 \ + --hash=sha256:a777c096a49d80f9d2979695b835b0f9c9edab73b59e4ceb51f19724dda887ed \ + --hash=sha256:a9a4ac9648d39ce71c2f63fe7dc6db144b9fa567ddfc48b9fde1b54483d26042 \ + --hash=sha256:aa4969f24d536ae2268c902b2c3d62ab464b5a66bcb247630d208a79a8098e9b \ + --hash=sha256:c7390f9b2119b2b43160abb34f63277a638504ef8df99f11cb52c1fda66a2e6f \ + --hash=sha256:e18e6ab84dfb0ab997faf8cca25a86ff15dfea4027b986322026cc99e0a892da +distro==1.5.0 \ + --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ + --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 +idna==2.10 \ + --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ + --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 +josepy==1.6.0 \ + --hash=sha256:0aab1c3ceffe045e7fd5bcfe7685e27e9d2758518d9ba7116b5de34087e70bf5 \ + --hash=sha256:65f077fc5902aca1e140ddb000e7abb081d5fb8421db60b6071076ef81c5bd27 +parsedatetime==2.6 \ + --hash=sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455 \ + --hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b +pyOpenSSL==20.0.1 \ + --hash=sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51 \ + --hash=sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b +pyRFC3339==1.1 \ + --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ + --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a +pycparser==2.20 \ + --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ + --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 +pyparsing==2.4.7 \ + --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ + --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b +python-augeas==0.5.0 \ + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 +pytz==2021.1 \ + --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \ + --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 +requests==2.25.1 \ + --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ + --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e +requests-toolbelt==0.9.1 \ + --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ + --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced +urllib3==1.26.3 \ + --hash=sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80 \ + --hash=sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73 +zope.component==4.6.2 \ + --hash=sha256:607628e4c84f7887a69a958542b5c304663e726b73aba0882e3a3f059bff14f3 \ + --hash=sha256:91628918218b3e6f6323de2a7b845e09ddc5cae131c034896c051b084bba3c92 +zope.deferredimport==4.3.1 \ + --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \ + --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a +zope.deprecation==4.4.0 \ + --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ + --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 +zope.event==4.5.0 \ + --hash=sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42 \ + --hash=sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330 +zope.hookable==5.0.1 \ + --hash=sha256:0194b9b9e7f614abba60c90b231908861036578297515d3d6508eb10190f266d \ + --hash=sha256:0c2977473918bdefc6fa8dfb311f154e7f13c6133957fe649704deca79b92093 \ + --hash=sha256:17b8bdb3b77e03a152ca0d5ca185a7ae0156f5e5a2dbddf538676633a1f7380f \ + --hash=sha256:29d07681a78042cdd15b268ae9decffed9ace68a53eebeb61d65ae931d158841 \ + --hash=sha256:36fb1b35d1150267cb0543a1ddd950c0bc2c75ed0e6e92e3aaa6ac2e29416cb7 \ + --hash=sha256:3aed60c2bb5e812bbf9295c70f25b17ac37c233f30447a96c67913ba5073642f \ + --hash=sha256:3cac1565cc768911e72ca9ec4ddf5c5109e1fef0104f19f06649cf1874943b60 \ + --hash=sha256:3d4bc0cc4a37c3cd3081063142eeb2125511db3c13f6dc932d899c512690378e \ + --hash=sha256:3f73096f27b8c28be53ffb6604f7b570fbbb82f273c6febe5f58119009b59898 \ + --hash=sha256:522d1153d93f2d48aa0bd9fb778d8d4500be2e4dcf86c3150768f0e3adbbc4ef \ + --hash=sha256:523d2928fb7377bbdbc9af9c0b14ad73e6eaf226349f105733bdae27efd15b5a \ + --hash=sha256:5848309d4fc5c02150a45e8f8d2227e5bfda386a508bbd3160fed7c633c5a2fa \ + --hash=sha256:6781f86e6d54a110980a76e761eb54590630fd2af2a17d7edf02a079d2646c1d \ + --hash=sha256:6fd27921ebf3aaa945fa25d790f1f2046204f24dba4946f82f5f0a442577c3e9 \ + --hash=sha256:70d581862863f6bf9e175e85c9d70c2d7155f53fb04dcdb2f73cf288ca559a53 \ + --hash=sha256:81867c23b0dc66c8366f351d00923f2bc5902820a24c2534dfd7bf01a5879963 \ + --hash=sha256:81db29edadcbb740cd2716c95a297893a546ed89db1bfe9110168732d7f0afdd \ + --hash=sha256:86bd12624068cea60860a0759af5e2c3adc89c12aef6f71cf12f577e28deefe3 \ + --hash=sha256:9c184d8f9f7a76e1ced99855ccf390ffdd0ec3765e5cbf7b9cada600accc0a1e \ + --hash=sha256:acc789e8c29c13555e43fe4bf9fcd15a65512c9645e97bbaa5602e3201252b02 \ + --hash=sha256:afaa740206b7660d4cc3b8f120426c85761f51379af7a5b05451f624ad12b0af \ + --hash=sha256:b5f5fa323f878bb16eae68ea1ba7f6c0419d4695d0248bed4b18f51d7ce5ab85 \ + --hash=sha256:bd89e0e2c67bf4ac3aca2a19702b1a37269fb1923827f68324ac2e7afd6e3406 \ + --hash=sha256:c212de743283ec0735db24ec6ad913758df3af1b7217550ff270038062afd6ae \ + --hash=sha256:ca553f524293a0bdea05e7f44c3e685e4b7b022cb37d87bc4a3efa0f86587a8d \ + --hash=sha256:cab67065a3db92f636128d3157cc5424a145f82d96fb47159c539132833a6d36 \ + --hash=sha256:d3b3b3eedfdbf6b02898216e85aa6baf50207f4378a2a6803d6d47650cd37031 \ + --hash=sha256:d9f4a5a72f40256b686d31c5c0b1fde503172307beb12c1568296e76118e402c \ + --hash=sha256:df5067d87aaa111ed5d050e1ee853ba284969497f91806efd42425f5348f1c06 \ + --hash=sha256:e2587644812c6138f05b8a41594a8337c6790e3baf9a01915e52438c13fc6bef \ + --hash=sha256:e27fd877662db94f897f3fd532ef211ca4901eb1a70ba456f15c0866a985464a \ + --hash=sha256:e427ebbdd223c72e06ba94c004bb04e996c84dec8a0fa84e837556ae145c439e \ + --hash=sha256:e583ad4309c203ef75a09d43434cf9c2b4fa247997ecb0dcad769982c39411c7 \ + --hash=sha256:e760b2bc8ece9200804f0c2b64d10147ecaf18455a2a90827fbec4c9d84f3ad5 \ + --hash=sha256:ea9a9cc8bcc70e18023f30fa2f53d11ae069572a162791224e60cd65df55fb69 \ + --hash=sha256:ecb3f17dce4803c1099bd21742cd126b59817a4e76a6544d31d2cca6e30dbffd \ + --hash=sha256:ed794e3b3de42486d30444fb60b5561e724ee8a2d1b17b0c2e0f81e3ddaf7a87 \ + --hash=sha256:ee885d347279e38226d0a437b6a932f207f691c502ee565aba27a7022f1285df \ + --hash=sha256:fd5e7bc5f24f7e3d490698f7b854659a9851da2187414617cd5ed360af7efd63 \ + --hash=sha256:fe45f6870f7588ac7b2763ff1ce98cce59369717afe70cc353ec5218bc854bcc +zope.interface==5.2.0 \ + --hash=sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1 \ + --hash=sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d \ + --hash=sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123 \ + --hash=sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232 \ + --hash=sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549 \ + --hash=sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102 \ + --hash=sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5 \ + --hash=sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45 \ + --hash=sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00 \ + --hash=sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc \ + --hash=sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7 \ + --hash=sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104 \ + --hash=sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034 \ + --hash=sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3 \ + --hash=sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3 \ + --hash=sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4 \ + --hash=sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86 \ + --hash=sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96 \ + --hash=sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546 \ + --hash=sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb \ + --hash=sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3 \ + --hash=sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b \ + --hash=sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b \ + --hash=sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec \ + --hash=sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae \ + --hash=sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e \ + --hash=sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386 \ + --hash=sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2 \ + --hash=sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a \ + --hash=sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d \ + --hash=sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a \ + --hash=sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24 \ + --hash=sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d \ + --hash=sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b \ + --hash=sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50 \ + --hash=sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523 \ + --hash=sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a \ + --hash=sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095 \ + --hash=sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a \ + --hash=sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520 \ + --hash=sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65 \ + --hash=sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11 \ + --hash=sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c \ + --hash=sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7 \ + --hash=sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332 \ + --hash=sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e \ + --hash=sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c \ + --hash=sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7 \ + --hash=sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20 \ + --hash=sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc \ + --hash=sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd \ + --hash=sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537 +zope.proxy==4.3.5 \ + --hash=sha256:00573dfa755d0703ab84bb23cb6ecf97bb683c34b340d4df76651f97b0bab068 \ + --hash=sha256:092049280f2848d2ba1b57b71fe04881762a220a97b65288bcb0968bb199ec30 \ + --hash=sha256:0cbd27b4d3718b5ec74fc65ffa53c78d34c65c6fd9411b8352d2a4f855220cf1 \ + --hash=sha256:17fc7e16d0c81f833a138818a30f366696653d521febc8e892858041c4d88785 \ + --hash=sha256:19577dfeb70e8a67249ba92c8ad20589a1a2d86a8d693647fa8385408a4c17b0 \ + --hash=sha256:207aa914576b1181597a1516e1b90599dc690c095343ae281b0772e44945e6a4 \ + --hash=sha256:219a7db5ed53e523eb4a4769f13105118b6d5b04ed169a283c9775af221e231f \ + --hash=sha256:2b50ea79849e46b5f4f2b0247a3687505d32d161eeb16a75f6f7e6cd81936e43 \ + --hash=sha256:5903d38362b6c716e66bbe470f190579c530a5baf03dbc8500e5c2357aa569a5 \ + --hash=sha256:5c24903675e271bd688c6e9e7df5775ac6b168feb87dbe0e4bcc90805f21b28f \ + --hash=sha256:5ef6bc5ed98139e084f4e91100f2b098a0cd3493d4e76f9d6b3f7b95d7ad0f06 \ + --hash=sha256:61b55ae3c23a126a788b33ffb18f37d6668e79a05e756588d9e4d4be7246ab1c \ + --hash=sha256:63ddb992931a5e616c87d3d89f5a58db086e617548005c7f9059fac68c03a5cc \ + --hash=sha256:6943da9c09870490dcfd50c4909c0cc19f434fa6948f61282dc9cb07bcf08160 \ + --hash=sha256:6ad40f85c1207803d581d5d75e9ea25327cd524925699a83dfc03bf8e4ba72b7 \ + --hash=sha256:6b44433a79bdd7af0e3337bd7bbcf53dd1f9b0fa66bf21bcb756060ce32a96c1 \ + --hash=sha256:6bbaa245015d933a4172395baad7874373f162955d73612f0b66b6c2c33b6366 \ + --hash=sha256:7007227f4ea85b40a2f5e5a244479f6a6dfcf906db9b55e812a814a8f0e2c28d \ + --hash=sha256:74884a0aec1f1609190ec8b34b5d58fb3b5353cf22b96161e13e0e835f13518f \ + --hash=sha256:7d25fe5571ddb16369054f54cdd883f23de9941476d97f2b92eb6d7d83afe22d \ + --hash=sha256:7e162bdc5e3baad26b2262240be7d2bab36991d85a6a556e48b9dfb402370261 \ + --hash=sha256:814d62678dc3a30f4aa081982d830b7c342cf230ffc9d030b020cb154eeebf9e \ + --hash=sha256:8878a34c5313ee52e20aa50b03138af8d472bae465710fb954d133a9bfd3c38d \ + --hash=sha256:a66a0d94e5b081d5d695e66d6667e91e74d79e273eee95c1747717ba9cb70792 \ + --hash=sha256:a69f5cbf4addcfdf03dda564a671040127a6b7c34cf9fe4973582e68441b63fa \ + --hash=sha256:b00f9f0c334d07709d3f73a7cb8ae63c6ca1a90c790a63b5e7effa666ef96021 \ + --hash=sha256:b6ed71e4a7b4690447b626f499d978aa13197a0e592950e5d7020308f6054698 \ + --hash=sha256:bdf5041e5851526e885af579d2f455348dba68d74f14a32781933569a327fddf \ + --hash=sha256:be034360dd34e62608419f86e799c97d389c10a0e677a25f236a971b2f40dac9 \ + --hash=sha256:cc8f590a5eed30b314ae6b0232d925519ade433f663de79cc3783e4b10d662ba \ + --hash=sha256:cd7a318a15fe6cc4584bf3c4426f092ed08c0fd012cf2a9173114234fe193e11 \ + --hash=sha256:cf19b5f63a59c20306e034e691402b02055c8f4e38bf6792c23cad489162a642 \ + --hash=sha256:cfc781ce442ec407c841e9aa51d0e1024f72b6ec34caa8fdb6ef9576d549acf2 \ + --hash=sha256:dea9f6f8633571e18bc20cad83603072e697103a567f4b0738d52dd0211b4527 \ + --hash=sha256:e4a86a1d5eb2cce83c5972b3930c7c1eac81ab3508464345e2b8e54f119d5505 \ + --hash=sha256:e7106374d4a74ed9ff00c46cc00f0a9f06a0775f8868e423f85d4464d2333679 \ + --hash=sha256:e98a8a585b5668aa9e34d10f7785abf9545fe72663b4bfc16c99a115185ae6a5 \ + --hash=sha256:f64840e68483316eb58d82c376ad3585ca995e69e33b230436de0cdddf7363f9 \ + --hash=sha256:f8f4b0a9e6683e43889852130595c8854d8ae237f2324a053cdd884de936aa9b \ + --hash=sha256:fc45a53219ed30a7f670a6d8c98527af0020e6fd4ee4c0a8fb59f147f06d816c diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 1d8001727..10308bd39 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,7 +1,7 @@ # Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in letsencrypt-auto's requirements file. # Some dev package versions specified here may be overridden by higher level constraints -# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). +# files during tests (eg. tools/certbot_constraints.txt). alabaster==0.7.10 apacheconfig==0.3.2 apipkg==1.4 @@ -16,8 +16,8 @@ backports.functools-lru-cache==1.5 backports.shutil-get-terminal-size==1.0.0 backports.ssl-match-hostname==3.7.0.1 bcrypt==3.1.6 -boto3==1.11.7 -botocore==1.14.7 +boto3==1.17.4 +botocore==1.20.4 cached-property==1.5.1 cloudflare==2.3.1 configparser==3.7.4 @@ -26,9 +26,9 @@ coverage==4.5.4 decorator==4.4.1 deprecated==1.2.10 dns-lexicon==3.3.17 -dnspython==1.15.0 -docker==3.7.2 -docker-compose==1.25.0 +dnspython==2.1.0 +docker==4.3.1 +docker-compose==1.26.2 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -85,7 +85,7 @@ pylint==2.4.3 # If pynsist version is upgraded, our NSIS template windows-installer/template.nsi # must be upgraded if necessary using the new built-in one from pynsist. pynacl==1.3.0 -pynsist==2.4 +pynsist==2.6 pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 @@ -94,8 +94,9 @@ pytest-sugar==0.9.2 pytest-rerunfailures==4.2 python-dateutil==2.8.1 python-digitalocean==1.11 -pywin32==227 -PyYAML==3.13 +python-dotenv==0.14.0 +pywin32==300 +PyYAML==5.3.1 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 requests-oauthlib==1.3.0 @@ -116,7 +117,7 @@ tox==3.14.0 tqdm==4.19.4 traitlets==4.3.3 twine==1.11.0 -typed-ast==1.4.0 +typed-ast==1.4.1 typing==3.6.4 uritemplate==3.0.0 virtualenv==16.6.2 diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index 02222008b..d2ebe3537 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -14,10 +14,6 @@ WORKDIR /opt/certbot # Copy certbot code COPY CHANGELOG.md README.rst src/ -# We keep the relative path to the requirements file the same because, as of -# writing this, tools/pip_install.py is used in the Dockerfile for Certbot -# plugins and this script expects to find the requirements file there. -COPY letsencrypt-auto-source/pieces/dependency-requirements.txt letsencrypt-auto-source/pieces/ COPY tools tools COPY acme src/acme COPY certbot src/certbot @@ -42,9 +38,7 @@ RUN apk add --no-cache --virtual .build-deps \ musl-dev \ libffi-dev \ && python tools/pipstrap.py \ - && pip install --no-build-isolation \ - -r letsencrypt-auto-source/pieces/dependency-requirements.txt \ - && pip install --no-build-isolation --no-cache-dir --no-deps \ - --editable src/acme \ - --editable src/certbot \ -&& apk del .build-deps + && python tools/pip_install.py --no-cache-dir \ + --editable src/acme \ + --editable src/certbot \ + && apk del .build-deps diff --git a/tools/docker/plugin/Dockerfile b/tools/docker/plugin/Dockerfile index 5a6673e5b..863efd105 100644 --- a/tools/docker/plugin/Dockerfile +++ b/tools/docker/plugin/Dockerfile @@ -11,4 +11,4 @@ COPY qemu-${QEMU_ARCH}-static /usr/bin/ COPY . /opt/certbot/src/plugin # Install the DNS plugin -RUN tools/pip_install.py --no-cache-dir --editable /opt/certbot/src/plugin +RUN python tools/pip_install.py --no-cache-dir --editable /opt/certbot/src/plugin diff --git a/tools/extract_changelog.py b/tools/extract_changelog.py index fb0b849aa..bd718191a 100755 --- a/tools/extract_changelog.py +++ b/tools/extract_changelog.py @@ -24,7 +24,7 @@ def main(): i = 0 while i < len(lines): if section_pattern.match(lines[i]): - i = i + 1 + i = i + 2 while i < len(lines): if NEW_SECTION_PATTERN.match(lines[i]): break @@ -32,8 +32,6 @@ def main(): i = i + 1 i = i + 1 - changelog = [entry for entry in changelog if entry] - print('\n'.join(changelog)) diff --git a/tools/finish_release.py b/tools/finish_release.py index bc8e832df..24e20987f 100755 --- a/tools/finish_release.py +++ b/tools/finish_release.py @@ -21,6 +21,7 @@ Run: python tools/finish_release.py ~/.ssh/githubpat.txt """ +import argparse import glob import os.path import re @@ -44,6 +45,34 @@ SNAPS = ['certbot'] + DNS_PLUGINS # for sanity checking. SNAP_ARCH_COUNT = 3 + +def parse_args(args): + """Parse command line arguments. + + :param args: command line arguments with the program name removed. This is + usually taken from sys.argv[1:]. + :type args: `list` of `str` + + :returns: parsed arguments + :rtype: argparse.Namespace + + """ + # Use the file's docstring for the help text and don't let argparse reformat it. + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('githubpat', help='path to your GitHub personal access token') + group = parser.add_mutually_exclusive_group() + # We use 'store_false' and a destination related to the other type of + # artifact to cause the flag being set to disable publishing of the other + # artifact. This makes using the parsed arguments later on a little simpler + # and cleaner. + group.add_argument('--snaps-only', action='store_false', dest='publish_windows', + help='Skip publishing other artifacts and only publish the snaps') + group.add_argument('--windows-only', action='store_false', dest='publish_snaps', + help='Skip publishing other artifacts and only publish the Windows installer') + return parser.parse_args(args) + + def download_azure_artifacts(tempdir): """Download and unzip build artifacts from Azure pipelines. @@ -181,8 +210,9 @@ def promote_snaps(version): def main(args): - github_access_token_file = args[0] + parsed_args = parse_args(args) + github_access_token_file = parsed_args.githubpat github_access_token = open(github_access_token_file, 'r').read().rstrip() with tempfile.TemporaryDirectory() as tempdir: @@ -191,8 +221,10 @@ def main(args): # again fails. Publishing the snaps can be done multiple times though # so we do that first to make it easier to run the script again later # if something goes wrong. - promote_snaps(version) - create_github_release(github_access_token, tempdir, version) + if parsed_args.publish_snaps: + promote_snaps(version) + if parsed_args.publish_windows: + create_github_release(github_access_token, tempdir, version) if __name__ == "__main__": main(sys.argv[1:]) diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 0b47fa5f8..e7a34286c 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -5,8 +5,6 @@ # set to 1, packages are installed using pinned versions of all of our # dependencies. See pip_install.py for more information on the versions pinned # to. -from __future__ import print_function - import os import re import subprocess diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index 0d41d12c4..44a61249b 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -1,12 +1,11 @@ #!/usr/bin/env python """Merges multiple Python requirements files into one file. -Requirements files specified later take precedence over earlier ones. Only -simple SomeProject==1.2.3 format is currently supported. +Requirements files specified later take precedence over earlier ones. +Only the simple formats SomeProject==1.2.3 or SomeProject<=1.2.3 are +currently supported. """ -from __future__ import print_function - import sys @@ -16,17 +15,28 @@ def process_entries(entries): :param list entries: List of entries - :returns: mapping from a project to its pinned version + :returns: mapping from a project to its version specifier :rtype: dict """ data = {} for e in entries: e = e.strip() if e and not e.startswith('#') and not e.startswith('-e'): - project, version = e.split('==') - if not version: + # Support for <= was added as part of + # https://github.com/certbot/certbot/pull/8460 because we weren't + # able to pin a package to an exact version. Normally, this + # functionality shouldn't be needed so we could remove it in the + # future. If you do so, make sure to update other places in this + # file related to this behavior such as this file's docstring. + for comparison in ('==', '<=',): + parts = e.split(comparison) + if len(parts) == 2: + project_name = parts[0] + version = parts[1] + data[project_name] = comparison + version + break + else: raise ValueError("Unexpected syntax '{0}'".format(e)) - data[project] = version return data def read_file(file_path): @@ -44,10 +54,11 @@ def read_file(file_path): def output_requirements(requirements): """Prepare print requirements to stdout. - :param dict requirements: mapping from a project to its pinned version + :param dict requirements: mapping from a project to its version + specifier """ - return '\n'.join('{0}=={1}'.format(key, value) + return '\n'.join('{0}{1}'.format(key, value) for key, value in sorted(requirements.items())) diff --git a/tools/offline-sigrequest.sh b/tools/offline-sigrequest.sh deleted file mode 100755 index 6443ae8af..000000000 --- a/tools/offline-sigrequest.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -set -o errexit - -function sayhash { # $1 <-- HASH ; $2 <---SIGFILEBALL - while read -p "Press Enter to read the hash aloud or type 'done': " INP && [ "$INP" = "" ] ; do - if ! `which festival > /dev/null` ; then - echo \`festival\` is not installed! - echo Please install it to read the hash aloud - else - cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \ - echo -n '(SayText "'; \ - sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ - echo '")' ) | festival - fi - done - - echo 'Paste in the data from the QR code, then type Ctrl-D:' - cat > $2 -} - -function offlinesign { # $1 <-- INPFILE ; $2 <---SIGFILE - echo HASH FOR SIGNING: - SIGFILEBALL="$2.lzma.base64" - #echo "(place the resulting raw binary signature in $SIGFILEBALL)" - sha256sum $1 - echo metahash for confirmation only $(sha256sum $1 |cut -d' ' -f1 | tr -d '\n' | sha256sum | cut -c1-6) ... - echo - sayhash $1 $SIGFILEBALL -} - -function oncesigned { # $1 <-- INPFILE ; $2 <--SIGFILE - SIGFILEBALL="$2.lzma.base64" - cat $SIGFILEBALL | tr -d '\r' | base64 -d | unlzma -c > $2 || exit 1 - if ! [ -f $2 ] ; then - echo "Failed to find $2"'!' - exit 1 - fi - - if file $2 | grep -qv " data" ; then - echo "WARNING WARNING $2 does not look like a binary signature:" - echo `file $2` - exit 1 - fi -} - -HERE=`dirname $0` -LEAUTO="`realpath $HERE`/../letsencrypt-auto-source/letsencrypt-auto" -SIGFILE="$LEAUTO".sig -offlinesign $LEAUTO $SIGFILE -oncesigned $LEAUTO $SIGFILE diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 5d1446005..f6528f396 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -1,76 +1,79 @@ -# This file contains the oldest versions of our dependencies we say we require -# in our packages or versions we need to support to maintain compatibility with -# the versions included in the various Linux distros where we are packaged. +# This file contains the oldest versions of our dependencies we're trying to +# support. Usually these version numbers are taken from the packages of our +# dependencies available in popular LTS Linux distros. Keeping compatibility +# with those versions makes it much easier for OS maintainers to update their +# Certbot packages. +# +# When updating these dependencies, we should try to only update them to the +# oldest version of the package that is found in a non-EOL'd version of +# CentOS, Debian, or Ubuntu that has Certbot packages in their OS repositories +# using a version of Python we support. If the distro is EOL'd or using a +# version of Python we don't support, it can be ignored. # CentOS/RHEL 7 EPEL constraints -cffi==1.6.0 +# Some of these constraints may be stricter than necessary because they +# initially referred to the Python 2 packages in CentOS/RHEL 7 with EPEL. +cffi==1.9.1 chardet==2.2.1 -configobj==4.7.2 ipaddress==1.0.16 mock==1.0.1 ndg-httpsclient==0.3.2 ply==3.4 +pyOpenSSL==17.3.0 pyasn1==0.1.9 pycparser==2.14 pyRFC3339==1.0 python-augeas==0.5.0 oauth2client==4.0.0 -six==1.9.0 -# setuptools 0.9.8 is the actual version packaged, but some other dependencies -# in this file require setuptools>=1.0 and there are no relevant changes for us -# between these versions. -setuptools==1.0.0 urllib3==1.10.2 zope.component==4.1.0 zope.event==4.0.3 zope.interface==4.0.5 # Debian Jessie Backports constraints -# Debian Jessie has reached end of life. However: -# When it becomes necessary to upgrade any of these dependencies, you should only update them to the oldest version of the package found -# in a non-EOL'd version of CentOS, Debian, or Ubuntu that has Certbot packages in their OS repositories. -PyICU==1.8 +# Debian Jessie has reached end of life so these dependencies can probably be +# updated as needed or desired. colorama==0.3.2 enum34==1.0.3 html5lib==0.999 -idna==2.0 pbr==1.8.0 pytz==2012rc0 # Debian Buster constraints google-api-python-client==1.5.5 +pyparsing==2.2.0 # Our setup.py constraints apacheconfig==0.3.2 cloudflare==1.5.1 -cryptography==1.2.3 -parsedatetime==1.3 -pyparsing==1.5.5 python-digitalocean==1.11 -requests[security]==2.6.0 +requests==2.6.0 # Ubuntu Xenial constraints +# Ubuntu Xenial only has versions of Python which we do not support available +# so these dependencies can probably be updated as needed or desired. ConfigArgParse==0.10.0 -pyOpenSSL==0.15.1 funcsigs==0.4 zope.hookable==4.0.4 # Ubuntu Bionic constraints. +cryptography==2.1.4 distro==1.0.1 # Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins # using their local-oldest-requirements.txt dns-lexicon==2.2.1 httplib2==0.9.2 +idna==2.6 +setuptools==39.0.1 +six==1.11.0 + +# Ubuntu Focal constraints +asn1crypto==0.24.0 +configobj==5.0.6 +parsedatetime==2.4 # Plugin constraints # These aren't necessarily the oldest versions we need to support # Tracking at https://github.com/certbot/certbot/issues/6473 boto3==1.4.7 botocore==1.7.41 - -# Old certbot[dev] constraints -# Old versions of certbot[dev] required ipdb and our normally pinned version of -# ipython which ipdb depends on doesn't support Python 2 so we pin an older -# version here to keep tests working while we have Python 2 support. -ipython==5.8.0 -prompt-toolkit==1.0.18 diff --git a/tools/pip_install.py b/tools/pip_install.py index f963e4660..e06650ff2 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -57,11 +57,15 @@ def certbot_oldest_processing(tools_path, args, test_constraints): def certbot_normal_processing(tools_path, test_constraints): repo_path = os.path.dirname(tools_path) certbot_requirements = os.path.normpath(os.path.join( - repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) + repo_path, 'tools/certbot_constraints.txt')) with open(certbot_requirements, 'r') as fd: - data = fd.readlines() + certbot_reqs = fd.readlines() + with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd: + pipstrap_reqs = fd.readlines() with open(test_constraints, 'w') as fd: - data = "\n".join(strip_hashes.process_entries(data)) + data_certbot = "\n".join(strip_hashes.process_entries(certbot_reqs)) + data_pipstrap = "\n".join(strip_hashes.process_entries(pipstrap_reqs)) + data = "\n".join([data_certbot, data_pipstrap]) fd.write(data) @@ -72,7 +76,7 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain # Here is the order by increasing priority: # 1) The general development constraints (tools/dev_constraints.txt) # 2) The general tests constraints (oldest_requirements.txt or - # certbot-auto's dependency-requirements.txt for the normal processing) + # certbot_constraints.txt + pipstrap's constraints for the normal processing) # 3) The local requirement file, typically local-oldest-requirement in oldest tests files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] if requirements: @@ -82,17 +86,18 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain fd.write(merged_requirements) -def call_with_print(command): +def call_with_print(command, env=None): + if not env: + env = os.environ print(command) - subprocess.check_call(command, shell=True) + subprocess.check_call(command, shell=True, env=env) -def pip_install_with_print(args_str, disable_build_isolation=True): - command = ['"', sys.executable, '" -m pip install --disable-pip-version-check '] - if disable_build_isolation: - command.append('--no-build-isolation ') - command.append(args_str) - call_with_print(''.join(command)) +def pip_install_with_print(args_str, env=None): + if not env: + env = os.environ + command = ['"', sys.executable, '" -m pip install --disable-pip-version-check ', args_str] + call_with_print(''.join(command), env=env) def main(args): @@ -113,20 +118,23 @@ def main(args): else: certbot_normal_processing(tools_path, test_constraints) + env = os.environ.copy() + env["PIP_CONSTRAINT"] = all_constraints + merge_requirements(tools_path, requirements, test_constraints, all_constraints) if requirements: # This branch is executed during the oldest tests # First step, install the transitive dependencies of oldest requirements # in respect with oldest constraints. - pip_install_with_print('--constraint "{0}" --requirement "{1}"' - .format(all_constraints, requirements)) + pip_install_with_print('--requirement "{0}"'.format(requirements), + env=env) # Second step, ensure that oldest requirements themselves are effectively # installed using --force-reinstall, and avoid corner cases like the one described # in https://github.com/certbot/certbot/issues/7014. pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"' .format(requirements)) - pip_install_with_print('--constraint "{0}" {1}'.format( - all_constraints, ' '.join(args))) + print(' '.join(args)) + pip_install_with_print(' '.join(args), env=env) if __name__ == '__main__': diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index abfe9f214..de2a0ff57 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -6,9 +6,6 @@ # https://github.com/pyca/cryptography/blob/a02fdd60d98273ca34427235c4ca96687a12b239/.travis/downstream.d/certbot.sh#L8-L9. # We should try to remember to keep their repo updated if we make any changes # to this script which may break things for them. - -from __future__ import absolute_import - import sys import pip_install diff --git a/tools/pipstrap.py b/tools/pipstrap.py index 2f21a9a5f..df3e46003 100755 --- a/tools/pipstrap.py +++ b/tools/pipstrap.py @@ -1,46 +1,16 @@ #!/usr/bin/env python """Uses pip to upgrade Python packaging tools to pinned versions.""" -from __future__ import absolute_import - import os -import shutil -import tempfile import pip_install -# We include the hashes of the packages here for extra verification of -# the packages downloaded from PyPI. This is especially valuable in our -# builds of Certbot that we ship to our users such as our Docker images. -# -# An older version of setuptools is currently used here in order to keep -# compatibility with Python 2 since newer versions of setuptools have dropped -# support for it. -REQUIREMENTS = r""" -pip==20.2.4 \ - --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \ - --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1 -setuptools==44.1.1 \ - --hash=sha256:27a714c09253134e60a6fa68130f78c7037e5562c4f21f8f318f2ae900d152d5 \ - --hash=sha256:c67aa55db532a0dadc4d2e20ba9961cbd3ccc84d544e9029699822542b5a476b -wheel==0.35.1 \ - --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \ - --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f -""" +_REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_constraints.txt") def main(): - with pip_install.temporary_directory() as tempdir: - requirements_filepath = os.path.join(tempdir, 'reqs.txt') - with open(requirements_filepath, 'w') as f: - f.write(REQUIREMENTS) - pip_install_args = '--requirement ' + requirements_filepath - # We don't disable build isolation because we may have an older - # version of pip that doesn't support the flag disabling it. We - # expect these packages to already have usable wheels available - # anyway so no building should be required. - pip_install.pip_install_with_print(pip_install_args, - disable_build_isolation=False) + pip_install_args = '--requirement "{0}"'.format(_REQUIREMENTS_PATH) + pip_install.pip_install_with_print(pip_install_args) if __name__ == '__main__': diff --git a/tools/pipstrap_constraints.txt b/tools/pipstrap_constraints.txt new file mode 100644 index 000000000..5de9e147d --- /dev/null +++ b/tools/pipstrap_constraints.txt @@ -0,0 +1,18 @@ +# Constraints for pipstrap.py +# +# We include the hashes of the packages here for extra verification of +# the packages downloaded from PyPI. This is especially valuable in our +# builds of Certbot that we ship to our users such as our Docker images. +# +# An older version of setuptools is currently used here in order to keep +# compatibility with Python 2 since newer versions of setuptools have dropped +# support for it. +pip==20.2.4 \ + --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \ + --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1 +setuptools==44.1.1 \ + --hash=sha256:27a714c09253134e60a6fa68130f78c7037e5562c4f21f8f318f2ae900d152d5 \ + --hash=sha256:c67aa55db532a0dadc4d2e20ba9961cbd3ccc84d544e9029699822542b5a476b +wheel==0.35.1 \ + --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \ + --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f diff --git a/tools/readlink.py b/tools/readlink.py index 446c8ebdc..c1e0c2e61 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -6,8 +6,6 @@ useful as there are often differences in readlink on different platforms. """ -from __future__ import print_function - import os import sys diff --git a/letsencrypt-auto-source/rebuild_dependencies.py b/tools/rebuild_certbot_constraints.py similarity index 84% rename from letsencrypt-auto-source/rebuild_dependencies.py rename to tools/rebuild_certbot_constraints.py index 864394661..f5e5d3ca7 100755 --- a/letsencrypt-auto-source/rebuild_dependencies.py +++ b/tools/rebuild_certbot_constraints.py @@ -4,12 +4,12 @@ Gather and consolidate the up-to-date dependencies available and required to ins on various Linux distributions. It generates a requirements file contained the pinned and hashed versions, ready to be used by pip to install the certbot dependencies. -This script is typically used to update the certbot-requirements.txt file of certbot-auto. +This script is typically used to update the certbot_constraints.txt file. To achieve its purpose, this script will start a certbot installation with unpinned dependencies, then gather them, on various distributions started as Docker containers. -Usage: letsencrypt-auto-source/rebuild_dependencies new_requirements.txt +Usage: tools/rebuild_certbot_constraints.py new_requirements.txt NB1: Docker must be installed on the machine running this script. NB2: Python library 'hashin' must be installed on the machine running this script. @@ -26,52 +26,41 @@ import argparse # The list of docker distributions to test dependencies against with. DISTRIBUTION_LIST = [ - 'ubuntu:18.04', 'ubuntu:16.04', - 'debian:stretch', - 'centos:7', 'centos:6', - 'opensuse/leap:15', - 'fedora:29', + 'ubuntu:20.04', 'ubuntu:18.04', 'debian:buster', + 'centos:8', 'centos:7', 'fedora:29', ] # These constraints will be added while gathering dependencies on each distribution. # It can be used because a particular version for a package is required for any reason, # or to solve a version conflict between two distributions requirements. AUTHORITATIVE_CONSTRAINTS = { - # Using an older version of mock here prevents regressions of #5276. - 'mock': '1.3.0', # Too touchy to move to a new version. And will be removed soon # in favor of pure python parser for Apache. 'python-augeas': '0.5.0', - # Package enum34 needs to be explicitly limited to Python2.x, in order to avoid - # certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456. - 'enum34': '1.1.10; python_version < \'3.4\'', - # Cryptography 2.9+ drops support for OpenSSL 1.0.1, but we still want to support it - # for officially supported non-x86_64 ancient distributions like RHEL 6. - 'cryptography': '2.8', - # Parsedatetime 2.6 is broken on Python 2.7, see https://github.com/bear/parsedatetime/issues/246 - 'parsedatetime': '2.5', + # We avoid cryptography 3.4+ since it requires Rust to compile the wheels, and + # this needs some work on the snap builds. + 'cryptography': '3.3.2', } -# ./certbot/letsencrypt-auto-source/rebuild_dependencies.py (2 levels from certbot root path) +# ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path) CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__))) # The script will be used to gather dependencies for a given distribution. -# - certbot-auto is used to install relevant OS packages, and set up an initial venv +# - bootstrap_os_packages.sh is used to install relevant OS packages, and set up an initial venv # - then this venv is used to consistently construct an empty new venv -# - once pipstraped, this new venv pip-installs certbot runtime (including apache/nginx), +# - once pipstrap.py, this new venv pip-installs certbot runtime (including apache/nginx), # without pinned dependencies, and respecting input authoritative requirements # - `certbot plugins` is called to check we have a healthy environment # - finally current set of dependencies is extracted out of the docker using pip freeze SCRIPT = r"""#!/bin/sh -set -e +set -ex cd /tmp/certbot -letsencrypt-auto-source/letsencrypt-auto --install-only -n -PYVER=`/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` +tests/letstest/scripts/bootstrap_os_packages.sh -/opt/eff.org/certbot/venv/bin/python letsencrypt-auto-source/pieces/create_venv.py /tmp/venv "$PYVER" 1 +python3 -m venv /tmp/venv -/tmp/venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py +/tmp/venv/bin/python tools/pipstrap.py /tmp/venv/bin/pip install -e acme -e certbot -e certbot-apache -e certbot-nginx -c /tmp/constraints.txt /tmp/venv/bin/certbot plugins /tmp/venv/bin/pip freeze >> /tmp/workspace/requirements.txt @@ -109,6 +98,7 @@ def _requirements_from_one_distribution(distribution, verbose): '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) command = ['docker', 'run', '--rm', '--cidfile', cid_file, + '--network=host', '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), '-v', '{0}:/tmp/workspace'.format(workspace), '-v', '{0}:/tmp/constraints.txt'.format(authoritative_constraints), @@ -158,7 +148,7 @@ def _parse_and_merge_requirements(dependencies_map, requirements_file_lines, dis """ for line in requirements_file_lines: match = re.match(r'([^=]+)==([^=]+)', line.strip()) - if not line.startswith('-e') and match: + if not line.startswith('-e') and not line.startswith('#') and match: package, version = match.groups() if package not in ['acme', 'certbot', 'certbot-apache', 'certbot-nginx', 'pkg-resources']: dependencies_map.setdefault(package, []).append((version, distribution)) @@ -215,11 +205,11 @@ def _write_requirements(dest_file, requirements, conflicts): print('===> Calculating hashes for the requirement file.') _write_to(dest_file, '''\ -# This is the flattened list of packages certbot-auto installs. +# This is the flattened list of pinned packages to build certbot deployable artifacts. # To generate this, do (with docker and package hashin installed): # ``` -# letsencrypt-auto-source/rebuild_dependencies.py \\ -# letsencrypt-auto-source/pieces/dependency-requirements.txt +# tools/rebuild_certbot_contraints.py \\ +# tools/certbot_constraints.txt # ``` # If you want to update a single dependency, run commands similar to these: # ``` @@ -264,8 +254,8 @@ def _gather_dependencies(dest_file, verbose): if __name__ == '__main__': parser = argparse.ArgumentParser( - description=('Build a sanitized, pinned and hashed requirements file for certbot-auto, ' - 'validated against several OS distributions using Docker.')) + description=('Build a sanitized, pinned and hashed requirements file for certbot deployable' + ' artifacts, validated against several OS distributions using Docker.')) parser.add_argument('requirements_path', help='path for the generated requirements file') parser.add_argument('--verbose', '-v', action='store_true', diff --git a/tools/run_oldest_tests.sh b/tools/run_oldest_tests.sh deleted file mode 100755 index 7bf9f2bc5..000000000 --- a/tools/run_oldest_tests.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -set -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -pushd "${DIR}/../" - -function cleanup() { - rm -f "${DOCKERFILE}" - popd -} - -trap cleanup EXIT - -DOCKERFILE=$(mktemp /tmp/Dockerfile.XXXXXX) - -cat << "EOF" >> "${DOCKERFILE}" -FROM ubuntu:16.04 -COPY letsencrypt-auto-source/pieces/dependency-requirements.txt /tmp/letsencrypt-auto-source/pieces/ -COPY tools/ /tmp/tools/ -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - python-dev python-pip python-setuptools \ - gcc libaugeas0 libssl-dev libffi-dev \ - git ca-certificates nginx-light openssl curl \ - && curl -fsSL https://get.docker.com | bash /dev/stdin \ - && python /tmp/tools/pipstrap.py \ - && python /tmp/tools/pip_install.py tox \ - && rm -rf /var/lib/apt/lists/* -EOF - -docker build -f "${DOCKERFILE}" -t oldest-worker . -docker run --rm --network=host -w "${PWD}" \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v "${PWD}:${PWD}" -v /tmp:/tmp \ - -e TOXENV -e ACME_SERVER -e PYTEST_ADDOPTS \ - oldest-worker python -m tox diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index 9666831d4..095fbc1d7 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -1,22 +1,22 @@ #!/usr/bin/env python3 import argparse -import glob import datetime -from multiprocessing import Pool, Process, Manager, Event +import glob import re import subprocess import sys -import tempfile +import time +from multiprocessing import Pool, Process, Manager from os.path import join, realpath, dirname, basename, exists - CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] def _execute_build(target, archs, status, workspace): process = subprocess.Popen([ - 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', '--build-on', ','.join(archs) + 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', + '--build-on', ','.join(archs) ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, cwd=workspace) process_output = [] @@ -31,7 +31,7 @@ def _execute_build(target, archs, status, workspace): return process.wait(), process_output -def _build_snap(target, archs, status, lock): +def _build_snap(target, archs, status, running, lock): status[target] = {arch: '...' for arch in archs} if target == 'certbot': @@ -43,7 +43,8 @@ def _build_snap(target, archs, status, lock): while retry: exit_code, process_output = _execute_build(target, archs, status, workspace) - print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with exit code {exit_code}.') + print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with ' + f'exit code {exit_code}.') sys.stdout.flush() with lock: @@ -53,7 +54,8 @@ def _build_snap(target, archs, status, lock): # We expect to have all target snaps available, or something bad happened. snaps_list = glob.glob(join(workspace, '*.snap')) if not len(snaps_list) == len(archs): - print(f'Some of the expected snaps for a successful build are missing (current list: {snaps_list}).') + print('Some of the expected snaps for a successful build are missing ' + f'(current list: {snaps_list}).') dump_output = True else: break @@ -67,9 +69,12 @@ def _build_snap(target, archs, status, lock): print(f'Dumping snapcraft remote-build output build for {target}:') print('\n'.join(process_output)) - # Retry the remote build if it has been interrupted (non zero status code) or if some builds have failed. + # Retry the remote build if it has been interrupted (non zero status code) + # or if some builds have failed. retry = retry - 1 + running[target] = False + return {target: workspace} @@ -100,15 +105,11 @@ def _dump_status_helper(archs, status): sys.stdout.flush() -def _dump_status(archs, status, stop_event): - while not stop_event.wait(10): - print('Remote build status at {0}'.format(datetime.datetime.now())) +def _dump_status(archs, status, running): + while any(running.values()): + print(f'Remote build status at {datetime.datetime.now()}') _dump_status_helper(archs, status) - - -def _dump_status_final(archs, status): - print('Results for remote build finished at {0}'.format(datetime.datetime.now())) - _dump_status_helper(archs, status) + time.sleep(10) def _dump_results(targets, archs, status, workspaces): @@ -124,10 +125,10 @@ def _dump_results(targets, archs, status, workspaces): if not exists(build_output_path): build_output = f'No output has been dumped by snapcraft remote-build.' else: - with open(join(workspaces[target], '{0}_{1}.txt'.format(target, arch))) as file_h: + with open(join(workspaces[target], f'{target}_{arch}.txt')) as file_h: build_output = file_h.read() - print('Output for failed build target={0} arch={1}'.format(target, arch)) + print(f'Output for failed build target={target} arch={arch}') print('-------------------------------------------') print(build_output) print('-------------------------------------------') @@ -138,6 +139,10 @@ def _dump_results(targets, archs, status, workspaces): else: print('Some builds failed.') + print() + print(f'Results for remote build finished at {datetime.datetime.now()}') + _dump_status_helper(archs, status) + return failures @@ -147,6 +152,8 @@ def main(): help='the list of snaps to build') parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default=['amd64'], help='the architectures for which snaps are built') + parser.add_argument('--timeout', type=int, default=None, + help='build process will fail after the provided timeout (in seconds)') args = parser.parse_args() archs = set(args.archs) @@ -162,7 +169,7 @@ def main(): # If we're building anything other than just Certbot, we need to # generate the snapcraft files for the DNS plugins. - if targets != set(('certbot',)): + if targets != {'certbot'}: subprocess.run(['tools/snap/generate_dnsplugins_all.sh'], check=True, cwd=CERTBOT_DIR) @@ -173,25 +180,29 @@ def main(): with Manager() as manager, Pool(processes=len(targets)) as pool: status = manager.dict() + running = manager.dict({target: True for target in targets}) lock = manager.Lock() - stop_event = Event() - state_process = Process(target=_dump_status, args=(archs, status, stop_event)) - state_process.start() + async_results = [pool.apply_async(_build_snap, (target, archs, status, running, lock)) + for target in targets] - async_results = [pool.apply_async(_build_snap, (target, archs, status, lock)) for target in targets] + process = Process(target=_dump_status, args=(archs, status, running)) + process.start() - workspaces = {} - for async_result in async_results: - workspaces.update(async_result.get()) + try: + process.join(args.timeout) - stop_event.set() - state_process.join() + if process.is_alive(): + raise ValueError(f"Timeout out reached ({args.timeout} seconds) during the build!") - failures = _dump_results(targets, archs, status, workspaces) - _dump_status_final(archs, status) + workspaces = {} + for async_result in async_results: + workspaces.update(async_result.get()) - return 1 if failures else 0 + if _dump_results(targets, archs, status, workspaces): + raise ValueError("There were failures during the build!") + finally: + process.terminate() if __name__ == '__main__': diff --git a/tools/snap/generate_dnsplugins_all.sh b/tools/snap/generate_dnsplugins_all.sh index 6c41a19cd..976b0dd7b 100755 --- a/tools/snap/generate_dnsplugins_all.sh +++ b/tools/snap/generate_dnsplugins_all.sh @@ -10,6 +10,7 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH # Create constraints file "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt) \ + <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_constraints.txt) \ + <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \ > "${PLUGIN_PATH}"/snap-constraints.txt done diff --git a/tools/snap/generate_dnsplugins_snapcraft.sh b/tools/snap/generate_dnsplugins_snapcraft.sh index 06807ec48..d93d8ec73 100755 --- a/tools/snap/generate_dnsplugins_snapcraft.sh +++ b/tools/snap/generate_dnsplugins_snapcraft.sh @@ -23,11 +23,16 @@ parts: ${PLUGIN}: plugin: python source: . - constraints: [\$SNAPCRAFT_PART_SRC/snap-constraints.txt] override-pull: | snapcraftctl pull snapcraftctl set-version \`grep ^version \$SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"\` build-environment: + # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the + # parts.[part_name].constraints option available in snapcraft.yaml when the Python plugin is + # used. This is done to let these constraints be applied not only on the certbot package + # build, but also on any isolated build that pip could trigger when building wheels for + # dependencies. See https://github.com/certbot/certbot/pull/8443 for more info. + - PIP_CONSTRAINT: \$SNAPCRAFT_PART_SRC/snap-constraints.txt - SNAP_BUILD: "True" # To build cryptography and cffi if needed build-packages: [gcc, libffi-dev, libssl-dev, python3-dev] diff --git a/tools/venv.py b/tools/venv.py index f99386eff..9f7488008 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -1,36 +1,261 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Developer virtualenv setup for Certbot client -import os -import sys +"""Aids in creating a developer virtual environment for Certbot. -import _venv_common +When this module is run as a script, it takes the arguments that should +be passed to pip to install the Certbot packages as command line +arguments. If no arguments are provided, all Certbot packages and their +development dependencies are installed. The virtual environment will be +created with the name "venv" in the current working directory. You can +change the name of the virtual environment by setting the environment +variable VENV_NAME. + +""" + +from __future__ import print_function + +from distutils.version import LooseVersion +import glob +import os +import re +import shutil +import subprocess +import sys +import time + +REQUIREMENTS = [ + '-e acme[dev]', + '-e certbot[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-compatibility-test', + '-e certbot-ci', +] + +VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') + + +class PythonExecutableNotFoundError(Exception): + pass + + +def find_python_executable() -> str: + """ + Find the relevant python executable that is of the given python major version. + Will test, in decreasing priority order: + + * the current Python interpreter + * 'pythonX' executable in PATH (with X the given major version) if available + * 'python' executable in PATH if available + * Windows Python launcher 'py' executable in PATH if available + + Incompatible python versions for Certbot will be evicted (e.g. Python 3 + versions less than 3.6). + + :rtype: str + :return: the relevant python executable path + :raise RuntimeError: if no relevant python executable path could be found + """ + python_executable_path = None + + # First try, current python executable + if _check_version('{0}.{1}.{2}'.format( + sys.version_info[0], sys.version_info[1], sys.version_info[2])): + return sys.executable + + # Second try, with python executables in path + for one_version in ('3', '',): + try: + one_python = 'python{0}'.format(one_version) + output = subprocess.check_output([one_python, '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + if _check_version(output.strip().split()[1]): + return subprocess.check_output([one_python, '-c', + 'import sys; sys.stdout.write(sys.executable);'], + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + pass + + # Last try, with Windows Python launcher + try: + output_version = subprocess.check_output(['py', '-3', '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + if _check_version(output_version.strip().split()[1]): + return subprocess.check_output(['py', env_arg, '-c', + 'import sys; sys.stdout.write(sys.executable);'], + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + pass + + if not python_executable_path: + raise RuntimeError('Error, no compatible Python executable for Certbot could be found.') + + +def _check_version(version_str): + search = VERSION_PATTERN.search(version_str) + + if not search: + return False + + version = (int(search.group(1)), int(search.group(2))) + + if version >= (3, 6): + return True + + print('Incompatible python version for Certbot found: {0}'.format(version_str)) + return False + + +def subprocess_with_print(cmd, env=None, shell=False): + if env is None: + env = os.environ + print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) + subprocess.check_call(cmd, env=env, shell=shell) + + +def subprocess_output_with_print(cmd, env=None, shell=False): + if env is None: + env = os.environ + print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) + return subprocess.check_output(cmd, env=env, shell=shell) + + +def get_venv_python_path(venv_path): + python_linux = os.path.join(venv_path, 'bin/python') + if os.path.isfile(python_linux): + return os.path.abspath(python_linux) + python_windows = os.path.join(venv_path, 'Scripts\\python.exe') + if os.path.isfile(python_windows): + return os.path.abspath(python_windows) + + raise ValueError(( + 'Error, could not find python executable in venv path {0}: is it a valid venv ?' + .format(venv_path))) + + +def prepare_venv_path(venv_name): + """Determines the venv path and prepares it for use. + + This function cleans up any Python eggs in the current working directory + and ensures the venv path is available for use. The path used is the + VENV_NAME environment variable if it is set and venv_name otherwise. If + there is already a directory at the desired path, the existing directory is + renamed by appending a timestamp to the directory name. + + :param str venv_name: The name or path at where the virtual + environment should be created if VENV_NAME isn't set. + + :returns: path where the virtual environment should be created + :rtype: str + + """ + for path in glob.glob('*.egg-info'): + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + env_venv_name = os.environ.get('VENV_NAME') + if env_venv_name: + print('Creating venv at {0}' + ' as specified in VENV_NAME'.format(env_venv_name)) + venv_name = env_venv_name + + if os.path.isdir(venv_name): + os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) + + return venv_name + + +def install_packages(venv_name, pip_args): + """Installs packages in the given venv. + + :param str venv_name: The name or path at where the virtual + environment should be created. + :param pip_args: Command line arguments that should be given to + pip to install packages + :type pip_args: `list` of `str` + + """ + # Using the python executable from venv, we ensure to execute following commands in this venv. + py_venv = get_venv_python_path(venv_name) + subprocess_with_print([py_venv, os.path.abspath('tools/pipstrap.py')]) + # We only use this value during pip install because: + # 1) We're really only adding it for installing cryptography, which happens here, and + # 2) There are issues with calling it along with VIRTUALENV_NO_DOWNLOAD, which applies at the + # steps above, not during pip install. + env_pip_no_binary = os.environ.get('CERTBOT_PIP_NO_BINARY') + if env_pip_no_binary: + # Check OpenSSL version. If it's too low, don't apply the env variable. + openssl_version_string = str(subprocess_output_with_print(['openssl', 'version'])) + matches = re.findall(r'OpenSSL ([^ ]+) ', openssl_version_string) + if not matches: + print('Could not find OpenSSL version, not setting PIP_NO_BINARY.') + else: + openssl_version = matches[0] + + if LooseVersion(openssl_version) >= LooseVersion('1.0.2'): + print('Setting PIP_NO_BINARY to {0}' + ' as specified in CERTBOT_PIP_NO_BINARY'.format(env_pip_no_binary)) + os.environ['PIP_NO_BINARY'] = env_pip_no_binary + else: + print('Not setting PIP_NO_BINARY, as OpenSSL version is too old.') + command = [py_venv, os.path.abspath('tools/pip_install.py')] + command.extend(pip_args) + subprocess_with_print(command) + if 'PIP_NO_BINARY' in os.environ: + del os.environ['PIP_NO_BINARY'] + + if os.path.isdir(os.path.join(venv_name, 'bin')): + # Linux/OSX specific + print('-------------------------------------------------------------------') + print('Please run the following command to activate developer environment:') + print('source {0}/bin/activate'.format(venv_name)) + print('-------------------------------------------------------------------') + elif os.path.isdir(os.path.join(venv_name, 'Scripts')): + # Windows specific + print('---------------------------------------------------------------------------') + print('Please run one of the following commands to activate developer environment:') + print('{0}\\Scripts\\activate.bat (for Batch)'.format(venv_name)) + print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name)) + print('---------------------------------------------------------------------------') + else: + raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) def create_venv(venv_path): - """Create a Python 2 virtual environment at venv_path. + """Create a Python virtual environment at venv_path. :param str venv_path: path where the venv should be created """ - python2 = _venv_common.find_python_executable(2) - command = [sys.executable, '-m', 'virtualenv', '--python', python2, venv_path] - - environ = os.environ.copy() - environ['VIRTUALENV_NO_DOWNLOAD'] = '1' - _venv_common.subprocess_with_print(command, environ) + python = find_python_executable() + command = [python, '-m', 'venv', venv_path] + subprocess_with_print(command) def main(pip_args=None): - if os.name == 'nt': - raise ValueError('Certbot for Windows is not supported on Python 2.x.') - - venv_path = _venv_common.prepare_venv_path('venv') + venv_path = prepare_venv_path('venv') create_venv(venv_path) if not pip_args: - pip_args = _venv_common.REQUIREMENTS + pip_args = REQUIREMENTS - _venv_common.install_packages(venv_path, pip_args) + install_packages(venv_path, pip_args) if __name__ == '__main__': diff --git a/tools/venv3.py b/tools/venv3.py deleted file mode 100755 index 7ead82bd5..000000000 --- a/tools/venv3.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python3 -# Developer virtualenv setup for Certbot client -import sys - -import _venv_common - - -def create_venv(venv_path): - """Create a Python 3 virtual environment at venv_path. - - :param str venv_path: path where the venv should be created - - """ - python3 = _venv_common.find_python_executable(3) - command = [python3, '-m', 'venv', venv_path] - _venv_common.subprocess_with_print(command) - - -def main(pip_args=None): - venv_path = _venv_common.prepare_venv_path('venv3') - create_venv(venv_path) - - if not pip_args: - pip_args = _venv_common.REQUIREMENTS + ['-e certbot[dev3]'] - - _venv_common.install_packages(venv_path, pip_args) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/tox.ini b/tox.ini index 5dcc55d3f..9f63b897c 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,7 @@ install_packages = source_paths = acme/acme certbot/certbot + certbot-ci/certbot_integration_tests certbot-apache/certbot_apache certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare @@ -76,49 +77,70 @@ setenv = PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto} PYTHONHASHSEED = 0 -[testenv:py27-oldest] +[testenv:oldest] +# Setting basepython allows the tests to fail fast if that version of Python +# isn't available instead of potentially trying to use a newer version of +# Python which is unlikely to work. +basepython = python3.6 commands = {[testenv]commands} setenv = {[testenv]setenv} CERTBOT_OLDEST=1 -[testenv:py27-acme-oldest] +[testenv:acme-oldest] +basepython = + {[testenv:oldest]basepython} commands = {[base]install_and_test} acme[dev] setenv = - {[testenv:py27-oldest]setenv} + {[testenv:oldest]setenv} -[testenv:py27-apache-oldest] +[testenv:apache-oldest] +basepython = + {[testenv:oldest]basepython} commands = {[base]install_and_test} certbot-apache setenv = - {[testenv:py27-oldest]setenv} + {[testenv:oldest]setenv} -[testenv:py27-apache-v2-oldest] +[testenv:apache-v2-oldest] +basepython = + {[testenv:oldest]basepython} commands = {[base]install_and_test} certbot-apache[dev] setenv = - {[testenv:py27-oldest]setenv} + {[testenv:oldest]setenv} -[testenv:py27-certbot-oldest] +[testenv:certbot-oldest] +basepython = + {[testenv:oldest]basepython} commands = {[base]install_and_test} certbot[dev] setenv = - {[testenv:py27-oldest]setenv} + {[testenv:oldest]setenv} -[testenv:py27-dns-oldest] +[testenv:dns-oldest] +basepython = + {[testenv:oldest]basepython} commands = {[base]install_and_test} {[base]dns_packages} setenv = - {[testenv:py27-oldest]setenv} + {[testenv:oldest]setenv} -[testenv:py27-nginx-oldest] +[testenv:nginx-oldest] +basepython = + {[testenv:oldest]basepython} commands = {[base]install_and_test} certbot-nginx python tests/lock_test.py setenv = - {[testenv:py27-oldest]setenv} + {[testenv:oldest]setenv} + +[testenv:external-mock] +commands = + python {toxinidir}/tools/pip_install.py mock + {[base]install_and_test} {[base]all_packages} [testenv:lint] basepython = python3 @@ -127,14 +149,12 @@ basepython = python3 # continue, but tox return code will reflect previous error commands = {[base]install_packages} - {[base]pip_install} certbot[dev3] python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} [testenv:mypy] basepython = python3 commands = {[base]install_packages} - {[base]pip_install} certbot[dev3] mypy {[base]source_paths} [testenv:apacheconftest] @@ -188,29 +208,6 @@ whitelist_externals = passenv = DOCKER_* -[testenv:le_auto_centos6] -# At the moment, this tests under Python 2.6 only, as only that version is -# readily available on the CentOS 6 Docker image. -commands = - python {toxinidir}/tests/modification-check.py - docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=centos -t lea letsencrypt-auto-source - docker run --rm -t lea -whitelist_externals = - docker -passenv = - DOCKER_* - TARGET_BRANCH - -[testenv:le_auto_oraclelinux6] -# At the moment, this tests under Python 2.6 only, as only that version is -# readily available on the Oracle Linux 6 Docker image. -commands = - docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=oraclelinux -t lea letsencrypt-auto-source - docker run --rm -t lea -whitelist_externals = - docker -passenv = DOCKER_* - [testenv:docker_dev] # tests the Dockerfile-dev file to ensure development with it works # as expected @@ -240,6 +237,17 @@ commands = --cov-config=certbot-ci/certbot_integration_tests/.coveragerc coverage report --include 'certbot/*' --show-missing --fail-under=62 +[testenv:integration-dns-rfc2136] +commands = + {[base]pip_install} acme certbot certbot-dns-rfc2136 certbot-ci + pytest certbot-ci/certbot_integration_tests/rfc2136_tests \ + --acme-server=pebble --dns-server=bind \ + --numprocesses=1 \ + --cov=acme --cov=certbot --cov=certbot_dns_rfc2136 --cov-report= \ + --cov-config=certbot-ci/certbot_integration_tests/.coveragerc + coverage report --include 'certbot/*' --show-missing --fail-under=45 + coverage report --include 'certbot-dns-rfc2136/*' --show-missing --fail-under=87 + [testenv:integration-external] # Run integration tests with Certbot outside of tox's virtual environment. commands = @@ -249,22 +257,26 @@ commands = passenv = DOCKER_* [testenv:integration-certbot-oldest] +basepython = + {[testenv:oldest]basepython} commands = {[base]pip_install} certbot {[base]pip_install} certbot-ci pytest certbot-ci/certbot_integration_tests/certbot_tests \ --acme-server={env:ACME_SERVER:pebble} passenv = DOCKER_* -setenv = {[testenv:py27-oldest]setenv} +setenv = {[testenv:oldest]setenv} [testenv:integration-nginx-oldest] +basepython = + {[testenv:oldest]basepython} commands = {[base]pip_install} certbot-nginx {[base]pip_install} certbot-ci pytest certbot-ci/certbot_integration_tests/nginx_tests \ --acme-server={env:ACME_SERVER:pebble} passenv = DOCKER_* -setenv = {[testenv:py27-oldest]setenv} +setenv = {[testenv:oldest]setenv} [testenv:test-farm-tests-base] changedir = tests/letstest diff --git a/windows-installer/construct.py b/windows-installer/construct.py index 14f770959..eb199a7e1 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -2,6 +2,7 @@ import contextlib import ctypes import os +import re import shutil import struct import subprocess @@ -9,10 +10,10 @@ import sys import tempfile import time -PYTHON_VERSION = (3, 7, 4) +PYTHON_VERSION = (3, 8, 8) PYTHON_BITNESS = 32 -PYWIN32_VERSION = 227 # do not forget to edit pywin32 dependency accordingly in setup.py -NSIS_VERSION = '3.04' +PYWIN32_VERSION = 300 # do not forget to edit pywin32 dependency accordingly in setup.py +NSIS_VERSION = '3.06.1' def main(): @@ -46,9 +47,26 @@ def _compile_wheels(repo_path, build_path, venv_python): wheels_project = [os.path.join(repo_path, package) for package in certbot_packages] with _prepare_constraints(repo_path) as constraints_file_path: - command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path, '--constraint', constraints_file_path] + env = os.environ.copy() + env['PIP_CONSTRAINT'] = constraints_file_path + command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path] command.extend(wheels_project) - subprocess.check_call(command) + subprocess.check_call(command, env=env) + + # Cryptography uses now a unique wheel name "cryptography-VERSION-cpXX-abi3-win32.whl where + # cpXX is the lowest supported version of Python (eg. cp36 says that the wheel is compatible + # with Python 3.6+). While technically valid to describe a wheel compliant with the Stable + # Application Binary Interface, this naming convention makes pynsist falsely think that the + # wheel is compatible with Python 3.6 only. + # Let's trick pynsist by renaming the wheel until this is fixed upstream. + for file in os.listdir(wheels_path): + # Given that our Python version is 3.8, this rename files like + # cryptography-VERSION-cpXX-abi3-win32.whl into cryptography-VERSION-cp38-abi3-win32.whl + renamed = re.sub(r'^(.*)-cp\d+-abi3-(\w+)\.whl$', r'\1-cp{0}{1}-abi3-\2.whl' + .format(PYTHON_VERSION[0], PYTHON_VERSION[1]), file) + print(renamed) + if renamed != file: + os.replace(os.path.join(wheels_path, file), os.path.join(wheels_path, renamed)) def _prepare_build_tools(venv_path, venv_python, repo_path): @@ -61,15 +79,20 @@ def _prepare_build_tools(venv_path, venv_python, repo_path): @contextlib.contextmanager def _prepare_constraints(repo_path): - requirements = os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'dependency-requirements.txt') - constraints = subprocess.check_output( - [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), requirements], + reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_constraints.txt') + reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt') + constraints_certbot = subprocess.check_output( + [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot], + universal_newlines=True) + constraints_pipstrap = subprocess.check_output( + [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_pipstrap], universal_newlines=True) workdir = tempfile.mkdtemp() try: constraints_file_path = os.path.join(workdir, 'constraints.txt') with open(constraints_file_path, 'a') as file_h: - file_h.write(constraints) + file_h.write(constraints_pipstrap) + file_h.write(constraints_certbot) file_h.write('pywin32=={0}'.format(PYWIN32_VERSION)) yield constraints_file_path finally: @@ -91,32 +114,6 @@ def _copy_assets(build_path, repo_path): def _generate_pynsist_config(repo_path, build_path): print('Generate pynsist configuration') - pywin32_paths_file = os.path.join(build_path, 'pywin32_paths.py') - - # Pywin32 uses non-standard folders to hold its packages. We need to instruct pynsist bootstrap - # explicitly to add them into sys.path. This is done with a custom "pywin32_paths.py" that is - # referred in the pynsist configuration as an "extra_preamble". - # Reference example: https://github.com/takluyver/pynsist/tree/master/examples/pywebview - with open(pywin32_paths_file, 'w') as file_h: - file_h.write('''\ -pkgdir = os.path.join(os.path.dirname(installdir), 'pkgs') - -sys.path.extend([ - os.path.join(pkgdir, 'win32'), - os.path.join(pkgdir, 'win32', 'lib'), -]) - -# Preload pywintypes and pythoncom -pwt = os.path.join(pkgdir, 'pywin32_system32', 'pywintypes{0}{1}.dll') -pcom = os.path.join(pkgdir, 'pywin32_system32', 'pythoncom{0}{1}.dll') -import warnings -with warnings.catch_warnings(): - warnings.simplefilter("ignore") - import imp -imp.load_dynamic('pywintypes', pwt) -imp.load_dynamic('pythoncom', pcom) -'''.format(PYTHON_VERSION[0], PYTHON_VERSION[1])) - installer_cfg_path = os.path.join(build_path, 'installer.cfg') certbot_pkg_path = os.path.join(repo_path, 'certbot') @@ -151,7 +148,6 @@ files=run.bat [Command certbot] entry_point=certbot.main:main -extra_preamble=pywin32_paths.py '''.format(certbot_version=certbot_version, installer_suffix='win_amd64' if PYTHON_BITNESS == 64 else 'win32', python_bitness=PYTHON_BITNESS, diff --git a/windows-installer/template.nsi b/windows-installer/template.nsi index 50a03865f..64bceb065 100644 --- a/windows-installer/template.nsi +++ b/windows-installer/template.nsi @@ -1,7 +1,7 @@ -; This NSIS template is based on the built-in one in pynsist 2.3. +; This NSIS template is based on the built-in one in pynsist 2.6. ; Added lines are enclosed within "CERTBOT CUSTOM BEGIN/END" comments. ; If pynsist is upgraded, this template must be updated if necessary using the new built-in one. -; Original file can be found here: https://github.com/takluyver/pynsist/blob/2.4/nsist/pyapp.nsi +; Original file can be found here: https://github.com/takluyver/pynsist/blob/2.6/nsist/pyapp.nsi !define PRODUCT_NAME "[[ib.appname]]" !define PRODUCT_VERSION "[[ib.version]]" @@ -14,9 +14,14 @@ ; Marker file to tell the uninstaller that it's a user installation !define USER_INSTALL_MARKER _user_install_marker - + SetCompressor lzma +!if "${NSIS_PACKEDVERSION}" >= 0x03000000 + Unicode true + ManifestDPIAware true +!endif + ; CERTBOT CUSTOM BEGIN ; Administrator privileges are required to insert a new task in Windows Scheduler. ; Also comment out some options to disable ability to choose AllUsers/CurrentUser install mode. @@ -35,9 +40,10 @@ SetCompressor lzma !define MULTIUSER_INSTALLMODE_FUNCTION correct_prog_files [% endif %] !include MultiUser.nsh +!include FileFunc.nsh [% block modernui %] -; Modern UI installer stuff +; Modern UI installer stuff !include "MUI2.nsh" !define MUI_ABORTWARNING !define MUI_ICON "[[icon]]" @@ -67,6 +73,8 @@ Name "${PRODUCT_NAME} (beta) ${PRODUCT_VERSION}" OutFile "${INSTALLER_NAME}" ShowInstDetails show +Var cmdLineInstallDir + Section -SETTINGS SetOutPath "$INSTDIR" SetOverwrite ifnewer @@ -96,14 +104,14 @@ Section "!${PRODUCT_NAME}" sec_app File "[[ file ]]" [% endfor %] [% endfor %] - + ; Install directories [% for dir, destination in ib.install_dirs %] SetOutPath "[[ pjoin(destination, dir) ]]" File /r "[[dir]]\*.*" [% endfor %] [% endblock install_files %] - + [% block install_shortcuts %] ; Install shortcuts ; The output path becomes the working directory for shortcuts @@ -127,7 +135,6 @@ Section "!${PRODUCT_NAME}" sec_app [% block install_commands %] [% if has_commands %] DetailPrint "Setting up command-line launchers..." - nsExec::ExecToLog '[[ python ]] -Es "$INSTDIR\_assemble_launchers.py" [[ python ]] "$INSTDIR\bin"' StrCmp $MultiUser.InstallMode CurrentUser 0 AddSysPathSystem ; Add to PATH for current user @@ -139,7 +146,7 @@ Section "!${PRODUCT_NAME}" sec_app AddedSysPath: [% endif %] [% endblock install_commands %] - + ; Byte-compile Python files. DetailPrint "Byte-compiling Python modules..." nsExec::ExecToLog '[[ python ]] -m compileall -q "$INSTDIR\pkgs"' @@ -238,12 +245,25 @@ Function .onMouseOverSection [% block mouseover_messages %] StrCmp $0 ${sec_app} "" +2 SendMessage $R0 ${WM_SETTEXT} 0 "STR:${PRODUCT_NAME}" - + [% endblock mouseover_messages %] FunctionEnd Function .onInit + ; Multiuser.nsh breaks /D command line parameter. Parse /INSTDIR instead. + ; Cribbing from https://nsis-dev.github.io/NSIS-Forums/html/t-299280.html + ${GetParameters} $0 + ClearErrors + ${GetOptions} '$0' "/INSTDIR=" $1 + IfErrors +2 ; Error means flag not found + StrCpy $cmdLineInstallDir $1 + ClearErrors + !insertmacro MULTIUSER_INIT + + ; If cmd line included /INSTDIR, override the install dir set by MultiUser + StrCmp $cmdLineInstallDir "" +2 + StrCpy $INSTDIR $cmdLineInstallDir FunctionEnd Function un.onInit @@ -257,4 +277,4 @@ Function correct_prog_files StrCmp $MultiUser.InstallMode AllUsers 0 +2 StrCpy $INSTDIR "$PROGRAMFILES64\${MULTIUSER_INSTALLMODE_INSTDIR}" FunctionEnd -[% endif %] \ No newline at end of file +[% endif %]