diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml index f835078c8..0c92136e8 100644 --- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml @@ -79,8 +79,6 @@ jobs: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.8 TOXENV: integration-dns-rfc2136 - le-auto-oraclelinux6: - TOXENV: le_auto_oraclelinux6 docker-dev: TOXENV: docker_dev macos-farmtest-apache2: diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index f0c6b6e49..900be9b2f 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -144,7 +144,7 @@ jobs: git config --global user.name "$(Build.RequestedFor)" mkdir -p ~/.local/share/snapcraft/provider/launchpad cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials - python3 tools/snap/build_remote.py ALL --archs ${ARCHS} + python3 tools/snap/build_remote.py ALL --archs ${ARCHS} --timeout 19800 displayName: Build snaps - script: | set -e diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index 897ab40bb..39cd628bc 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -58,9 +58,9 @@ jobs: apache-compat: IMAGE_NAME: ubuntu-18.04 TOXENV: apache_compat - le-auto-centos6: + le-modification: IMAGE_NAME: ubuntu-18.04 - TOXENV: le_auto_centos6 + TOXENV: modification apacheconftest: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 2.7 diff --git a/AUTHORS.md b/AUTHORS.md index f76c323a5..ff5c61613 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -149,6 +149,7 @@ Authors * [Lior Sabag](https://github.com/liorsbg) * [Lipis](https://github.com/lipis) * [lord63](https://github.com/lord63) +* [Lorenzo Fundaró](https://github.com/lfundaro) * [Luca Beltrame](https://github.com/lbeltrame) * [Luca Ebach](https://github.com/lucebac) * [Luca Olivetti](https://github.com/olivluca) diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index d1679fcad..3ec5203bf 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -20,3 +20,10 @@ for mod in list(sys.modules): # preserved (acme.jose.* is josepy.*) if mod == 'josepy' or mod.startswith('josepy.'): sys.modules['acme.' + mod.replace('josepy', 'jose', 1)] = sys.modules[mod] + +if sys.version_info[0] == 2: + warnings.warn( + "Python 2 support will be dropped in the next release of acme. " + "Please upgrade your Python version.", + PendingDeprecationWarning, + ) # pragma: no cover diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index f8b7e2b30..cabc7f4d1 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -186,6 +186,7 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu raise errors.Error(error) return client_ssl.get_peer_certificate() + def make_csr(private_key_pem, domains, must_staple=False): """Generate a CSR containing a list of domains as subjectAltNames. @@ -217,6 +218,7 @@ def make_csr(private_key_pem, domains, must_staple=False): return crypto.dump_certificate_request( crypto.FILETYPE_PEM, csr) + def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): common_name = loaded_cert_or_req.get_subject().CN sans = _pyopenssl_cert_or_req_san(loaded_cert_or_req) @@ -225,6 +227,7 @@ def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): return sans return [common_name] + [d for d in sans if d != common_name] + def _pyopenssl_cert_or_req_san(cert_or_req): """Get Subject Alternative Names from certificate or CSR using pyOpenSSL. @@ -317,6 +320,7 @@ def gen_ss_cert(key, domains, not_before=None, cert.sign(key, "sha256") return cert + def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. diff --git a/acme/setup.py b/acme/setup.py index 55d1ae17d..f8f9efaad 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index c20a4fdd6..16def1998 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -327,6 +327,9 @@ class ApacheConfigurator(common.Installer): if self.version < (2, 2): raise errors.NotSupportedError( "Apache Version {0} not supported.".format(str(self.version))) + elif self.version < (2, 4): + logger.warning('Support for Apache 2.2 is deprecated and will be removed in a ' + 'future release.') # Recover from previous crash before Augeas initialization to have the # correct parse tree from the get go. diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index c6fadb929..8b908ade7 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 2a0cda9b3..ee12d4706 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.9.0" +LE_AUTO_VERSION="1.10.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -799,11 +799,7 @@ BootstrapMageiaCommon() { # that function. If Bootstrap is set to a function that doesn't install any # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then - Bootstrap() { - BootstrapMessage "Debian-based OSes" - BootstrapDebCommon - } - BOOTSTRAP_VERSION="BootstrapDebCommon $BOOTSTRAP_DEB_COMMON_VERSION" + DEPRECATED_OS=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 @@ -1497,18 +1493,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.9.0 \ - --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \ - --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57 -acme==1.9.0 \ - --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \ - --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f -certbot-apache==1.9.0 \ - --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \ - --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e -certbot-nginx==1.9.0 \ - --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \ - --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65 +certbot==1.10.1 \ + --hash=sha256:011ac980fa21b9f29e02c9b8d8b86e8a4bf4670b51b6ad91656e401e9d2d2231 \ + --hash=sha256:0d9ee3fc09e0d03b2d1b1f1c4916e61ecfc6904b4216ddef4e6a5ca1424d9cb7 +acme==1.10.1 \ + --hash=sha256:752d598e54e98ad1e874de53fd50c61044f1b566d6deb790db5676ce9c573546 \ + --hash=sha256:fcbb559aedc96b404edf593e78517dcd7291984d5a37036c3fc77f3c5c122fd8 +certbot-apache==1.10.1 \ + --hash=sha256:f077b4b7f166627ef5e0921fe7cde57700670fc86e9ad9dbdfaf2c573cc0f2fa \ + --hash=sha256:97ed637b4c7b03820db6c69aa90145dc989933351d46a3d62baf6b71674f0a10 +certbot-nginx==1.10.1 \ + --hash=sha256:7c36459021f8a1ec3b6c062e4c4fc866bfaa1dbf26ccd29e043dd6848003be08 \ + --hash=sha256:c0bbeccf85f46b728fd95e6bb8c2649d32d3383d7f47ea4b9c312d12bf04d2f0 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py b/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py index 60c2fcdd8..819cb3e78 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py @@ -1,3 +1,4 @@ +# pylint: disable=missing-module-docstring import pytest # Custom assertions defined in the following package need to be registered to be properly diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index e295aefd7..b9854b402 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -77,6 +77,6 @@ class IntegrationTestsContext(object): appending the pytest worker id to the subdomain, using this pattern: {subdomain}.{worker_id}.wtf :param subdomain: the subdomain to use in the generated domain (default 'le') - :return: the well-formed domain suitable for redirection on + :return: the well-formed domain suitable for redirection on """ return '{0}.{1}.wtf'.format(subdomain, self.worker_id) diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index a91819180..546f96305 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -29,8 +29,9 @@ from certbot_integration_tests.certbot_tests.assertions import EVERYBODY_SID from certbot_integration_tests.utils import misc -@pytest.fixture() -def context(request): +@pytest.fixture(name='context') +def test_context(request): + # pylint: disable=missing-function-docstring # Fixture request is a built-in pytest fixture describing current test request. integration_test_context = certbot_context.IntegrationTestsContext(request) try: @@ -222,14 +223,16 @@ def test_renew_files_propagate_permissions(context): if os.name != 'nt': os.chmod(privkey1, 0o444) else: - import win32security - import ntsecuritycon + import win32security # pylint: disable=import-error + import ntsecuritycon # pylint: disable=import-error # Get the current DACL of the private key security = win32security.GetFileSecurity(privkey1, win32security.DACL_SECURITY_INFORMATION) dacl = security.GetSecurityDescriptorDacl() # Create a read permission for Everybody group everybody = win32security.ConvertStringSidToSid(EVERYBODY_SID) - dacl.AddAccessAllowedAce(win32security.ACL_REVISION, ntsecuritycon.FILE_GENERIC_READ, everybody) + dacl.AddAccessAllowedAce( + win32security.ACL_REVISION, ntsecuritycon.FILE_GENERIC_READ, everybody + ) # Apply the updated DACL to the private key security.SetSecurityDescriptorDacl(1, dacl, 0) win32security.SetFileSecurity(privkey1, win32security.DACL_SECURITY_INFORMATION, security) @@ -238,12 +241,14 @@ def test_renew_files_propagate_permissions(context): assert_cert_count_for_lineage(context.config_dir, certname, 2) if os.name != 'nt': - # On Linux, read world permissions + all group permissions will be copied from the previous private key + # On Linux, read world permissions + all group permissions + # will be copied from the previous private key assert_world_read_permissions(privkey2) assert_equals_world_read_permissions(privkey1, privkey2) assert_equals_group_permissions(privkey1, privkey2) else: - # On Windows, world will never have any permissions, and group permission is irrelevant for this platform + # On Windows, world will never have any permissions, and + # group permission is irrelevant for this platform assert_world_no_permissions(privkey2) @@ -609,19 +614,22 @@ def test_revoke_multiple_lineages(context): with open(join(context.config_dir, 'renewal', '{0}.conf'.format(cert2)), 'r') as file: data = file.read() - data = re.sub('archive_dir = .*\n', - 'archive_dir = {0}\n'.format(join(context.config_dir, 'archive', cert1).replace('\\', '\\\\')), - data) + data = re.sub( + 'archive_dir = .*\n', + 'archive_dir = {0}\n'.format( + join(context.config_dir, 'archive', cert1).replace('\\', '\\\\') + ), data + ) with open(join(context.config_dir, 'renewal', '{0}.conf'.format(cert2)), 'w') as file: file.write(data) - output = context.certbot([ + context.certbot([ 'revoke', '--cert-path', join(context.config_dir, 'live', cert1, 'cert.pem') ]) with open(join(context.workspace, 'logs', 'letsencrypt.log'), 'r') as f: - assert 'Not deleting revoked certs due to overlapping archive dirs' in f.read() + assert 'Not deleting revoked certificates due to overlapping archive dirs' in f.read() def test_wildcard_certificates(context): diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index bb5c07dac..230fb0eda 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -13,7 +13,6 @@ import sys from certbot_integration_tests.utils import acme_server as acme_lib from certbot_integration_tests.utils import dns_server as dns_lib -from certbot_integration_tests.utils.dns_server import DNSServer def pytest_addoption(parser): @@ -92,8 +91,10 @@ def _setup_primary_node(config): try: subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT) except (subprocess.CalledProcessError, OSError): - raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, ' - 'but is not installed or not available for current user.') + raise ValueError( + 'Error: docker-compose is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.' + ) # Parameter numprocesses is added to option by pytest-xdist workers = ['primary'] if not config.option.numprocesses\ diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/context.py b/certbot-ci/certbot_integration_tests/nginx_tests/context.py index 3a769840c..6f0f833a0 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/context.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/context.py @@ -1,3 +1,4 @@ +"""Module to handle the context of nginx integration tests.""" import os import subprocess diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py b/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py index 18991ae62..bbbe8ea06 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """General purpose nginx test configuration generator.""" import getpass @@ -42,6 +43,8 @@ events {{ worker_connections 1024; }} +# “This comment contains valid Unicode”. + http {{ # Set an array of temp, cache and log file options that will otherwise default to # restricted locations accessible only to root. @@ -51,61 +54,61 @@ http {{ #scgi_temp_path {nginx_root}/scgi_temp; #uwsgi_temp_path {nginx_root}/uwsgi_temp; access_log {nginx_root}/error.log; - + # This should be turned off in a Virtualbox VM, as it can cause some # interesting issues with data corruption in delivered files. sendfile off; - + tcp_nopush on; tcp_nodelay on; keepalive_timeout 65; types_hash_max_size 2048; - + #include /etc/nginx/mime.types; index index.html index.htm index.php; - + log_format main '$remote_addr - $remote_user [$time_local] $status ' '"$request" $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; - + default_type application/octet-stream; - + server {{ # IPv4. listen {http_port} {default_server}; # IPv6. listen [::]:{http_port} {default_server}; server_name nginx.{wtf_prefix}.wtf nginx2.{wtf_prefix}.wtf; - + root {nginx_webroot}; - + location / {{ # First attempt to serve request as file, then as directory, then fall # back to index.html. try_files $uri $uri/ /index.html; }} }} - + server {{ listen {http_port}; listen [::]:{http_port}; server_name nginx3.{wtf_prefix}.wtf; - + root {nginx_webroot}; - + location /.well-known/ {{ return 404; }} - + return 301 https://$host$request_uri; }} - + server {{ listen {other_port}; listen [::]:{other_port}; server_name nginx4.{wtf_prefix}.wtf nginx5.{wtf_prefix}.wtf; }} - + server {{ listen {http_port}; listen [::]:{http_port}; diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py index 1a62ea8d7..8a2d48a50 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -2,13 +2,14 @@ import os import ssl +from typing import List import pytest from certbot_integration_tests.nginx_tests import context as nginx_context -@pytest.fixture() -def context(request): +@pytest.fixture(name='context') +def test_context(request): # Fixture request is a built-in pytest fixture describing current test request. integration_test_context = nginx_context.IntegrationTestsContext(request) try: @@ -27,10 +28,12 @@ def context(request): # No matching server block; default_server does not exist ('nginx5.{0}.wtf', ['--preferred-challenges', 'http'], {'default_server': False}), # Multiple domains, mix of matching and not - ('nginx6.{0}.wtf,nginx7.{0}.wtf', ['--preferred-challenges', 'http'], {'default_server': False}), + ('nginx6.{0}.wtf,nginx7.{0}.wtf', [ + '--preferred-challenges', 'http' + ], {'default_server': False}), ], indirect=['context']) def test_certificate_deployment(certname_pattern, params, context): - # type: (str, list, nginx_context.IntegrationTestsContext) -> None + # type: (str, List[str], nginx_context.IntegrationTestsContext) -> None """ Test various scenarios to deploy a certificate to nginx using certbot. """ @@ -41,7 +44,9 @@ def test_certificate_deployment(certname_pattern, params, context): lineage = domains.split(',')[0] server_cert = ssl.get_server_certificate(('localhost', context.tls_alpn_01_port)) - with open(os.path.join(context.workspace, 'conf/live/{0}/cert.pem'.format(lineage)), 'r') as file: + with open(os.path.join( + context.workspace, 'conf/live/{0}/cert.pem'.format(lineage)), 'r' + ) as file: certbot_cert = file.read() assert server_cert == certbot_cert diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py index b9fe8b401..bdedee1fe 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -1,7 +1,10 @@ -from contextlib import contextmanager -from pytest import skip -from pkg_resources import resource_filename +"""Module to handle the context of RFC2136 integration tests.""" + import tempfile +from contextlib import contextmanager + +from pkg_resources import resource_filename +from pytest import skip from certbot_integration_tests.certbot_tests import context as certbot_context from certbot_integration_tests.utils import certbot_call @@ -33,7 +36,6 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): @contextmanager def rfc2136_credentials(self, label='default'): - # type: (str) -> str """ Produces the contents of a certbot-dns-rfc2136 credentials file. :param str label: which RFC2136 credential to use @@ -52,10 +54,10 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): ) with tempfile.NamedTemporaryFile('w+', prefix='rfc2136-creds-{}'.format(label), - suffix='.ini', dir=self.workspace) as f: - f.write(contents) - f.flush() - yield f.name + suffix='.ini', dir=self.workspace) as fp: + fp.write(contents) + fp.flush() + yield fp.name def skip_if_no_bind9_server(self): """Skips the test if there was no RFC2136-capable DNS server configured diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/test_main.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/test_main.py index 69996d533..ae6c0018e 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/test_main.py @@ -4,8 +4,9 @@ import pytest from certbot_integration_tests.rfc2136_tests import context as rfc2136_context -@pytest.fixture() -def context(request): +@pytest.fixture(name="context") +def pytest_context(request): + # pylint: disable=missing-function-docstring # Fixture request is a built-in pytest fixture describing current test request. integration_test_context = rfc2136_context.IntegrationTestsContext(request) try: diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 5559b44a6..bbbdd196b 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -7,18 +7,19 @@ import errno import json import os from os.path import join -import re import shutil import subprocess import sys import tempfile import time +from typing import List import requests from certbot_integration_tests.utils import misc from certbot_integration_tests.utils import pebble_artifacts from certbot_integration_tests.utils import proxy +# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils.constants import * @@ -31,10 +32,11 @@ class ACMEServer(object): ACMEServer gives access the acme_xdist parameter, listing the ports and directory url to use for each pytest node. It exposes also start and stop methods in order to start the stack, and stop it with proper resources cleanup. - ACMEServer is also a context manager, and so can be used to ensure ACME server is started/stopped - upon context enter/exit. + ACMEServer is also a context manager, and so can be used to ensure ACME server is + started/stopped upon context enter/exit. """ - def __init__(self, acme_server, nodes, http_proxy=True, stdout=False, dns_server=None): + def __init__(self, acme_server, nodes, http_proxy=True, stdout=False, + dns_server=None, http_01_port=DEFAULT_HTTP_01_PORT): """ Create an ACMEServer instance. :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) @@ -42,15 +44,22 @@ class ACMEServer(object): :param bool http_proxy: if False do not start the HTTP proxy :param bool stdout: if True stream all subprocesses stdout to standard stdout :param str dns_server: if set, Pebble/Boulder will use it to resolve domains + :param int http_01_port: port to use for http-01 validation; currently + only supported for pebble without an HTTP proxy """ self._construct_acme_xdist(acme_server, nodes) self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' self._proxy = http_proxy self._workspace = tempfile.mkdtemp() - self._processes = [] + self._processes = [] # type: List[subprocess.Popen] self._stdout = sys.stdout if stdout else open(os.devnull, 'w') self._dns_server = dns_server + self._http_01_port = http_01_port + if http_01_port != DEFAULT_HTTP_01_PORT: + if self._acme_type != 'pebble' or self._proxy: + raise ValueError('setting http_01_port is not currently supported ' + 'with boulder or the HTTP proxy') def start(self): """Start the test stack""" @@ -107,26 +116,34 @@ class ACMEServer(object): """Generate and return the acme_xdist dict""" acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT} - # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble. + # Directory and ACME port are set implicitly in the docker-compose.yml + # files of Boulder/Pebble. if acme_server == 'pebble': acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL else: # boulder acme_xdist['directory_url'] = BOULDER_V2_DIRECTORY_URL \ if acme_server == 'boulder-v2' else BOULDER_V1_DIRECTORY_URL - acme_xdist['http_port'] = {node: port for (node, port) - in zip(nodes, range(5200, 5200 + len(nodes)))} - acme_xdist['https_port'] = {node: port for (node, port) - in zip(nodes, range(5100, 5100 + len(nodes)))} - acme_xdist['other_port'] = {node: port for (node, port) - in zip(nodes, range(5300, 5300 + len(nodes)))} + acme_xdist['http_port'] = { + node: port for (node, port) in # pylint: disable=unnecessary-comprehension + zip(nodes, range(5200, 5200 + len(nodes))) + } + acme_xdist['https_port'] = { + node: port for (node, port) in # pylint: disable=unnecessary-comprehension + zip(nodes, range(5100, 5100 + len(nodes))) + } + acme_xdist['other_port'] = { + node: port for (node, port) in # pylint: disable=unnecessary-comprehension + zip(nodes, range(5300, 5300 + len(nodes))) + } self.acme_xdist = acme_xdist def _prepare_pebble_server(self): """Configure and launch the Pebble server""" print('=> Starting pebble instance deployment...') - pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts.fetch(self._workspace) + pebble_artifacts_rv = pebble_artifacts.fetch(self._workspace, self._http_01_port) + pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts_rv # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment. @@ -149,10 +166,10 @@ class ACMEServer(object): [pebble_path, '-config', pebble_config_path, '-dnsserver', dns_server, '-strict'], env=environ) - # pebble_ocsp_server is imported here and not at the top of module in order to avoid a useless - # ImportError, in the case where cryptography dependency is too old to support ocsp, but - # Boulder is used instead of Pebble, so pebble_ocsp_server is not used. This is the typical - # situation of integration-certbot-oldest tox testenv. + # pebble_ocsp_server is imported here and not at the top of module in order to avoid a + # useless ImportError, in the case where cryptography dependency is too old to support + # ocsp, but Boulder is used instead of Pebble, so pebble_ocsp_server is not used. This is + # the typical situation of integration-certbot-oldest tox testenv. from certbot_integration_tests.utils import pebble_ocsp_server self._launch_process([sys.executable, pebble_ocsp_server.__file__]) @@ -178,11 +195,12 @@ class ACMEServer(object): if self._dns_server: # Change Boulder config to use the provided DNS server - with open(join(instance_path, 'test/config/va.json'), 'r') as file_h: - config = json.loads(file_h.read()) - config['va']['dnsResolvers'] = [self._dns_server] - with open(join(instance_path, 'test/config/va.json'), 'w') as file_h: - file_h.write(json.dumps(config, indent=2, separators=(',', ': '))) + for suffix in ["", "-remote-a", "-remote-b"]: + with open(join(instance_path, 'test/config/va{}.json'.format(suffix)), 'r') as f: + config = json.loads(f.read()) + config['va']['dnsResolvers'] = [self._dns_server] + with open(join(instance_path, 'test/config/va{}.json'.format(suffix)), 'w') as f: + f.write(json.dumps(config, indent=2, separators=(',', ': '))) try: # Launch the Boulder server @@ -194,13 +212,16 @@ class ACMEServer(object): if not self._dns_server: # Configure challtestsrv to answer any A record request with ip of the docker host. - response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), - json={'ip': '10.77.77.1'}) + response = requests.post('http://localhost:{0}/set-default-ipv4'.format( + CHALLTESTSRV_PORT), json={'ip': '10.77.77.1'} + ) response.raise_for_status() except BaseException: # If we failed to set up boulder, print its logs. print('=> Boulder setup failed. Boulder logs are:') - process = self._launch_process(['docker-compose', 'logs'], cwd=instance_path, force_stderr=True) + process = self._launch_process([ + 'docker-compose', 'logs'], cwd=instance_path, force_stderr=True + ) process.wait() raise @@ -211,7 +232,7 @@ class ACMEServer(object): print('=> Configuring the HTTP proxy...') mapping = {r'.+\.{0}\.wtf'.format(node): 'http://127.0.0.1:{0}'.format(port) for node, port in self.acme_xdist['http_port'].items()} - command = [sys.executable, proxy.__file__, str(HTTP_01_PORT), json.dumps(mapping)] + command = [sys.executable, proxy.__file__, str(DEFAULT_HTTP_01_PORT), json.dumps(mapping)] self._launch_process(command) print('=> Finished configuring the HTTP proxy.') @@ -220,12 +241,15 @@ class ACMEServer(object): if not env: env = os.environ stdout = sys.stderr if force_stderr else self._stdout - process = subprocess.Popen(command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env) + process = subprocess.Popen( + command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env + ) self._processes.append(process) return process def main(): + # pylint: disable=missing-function-docstring parser = argparse.ArgumentParser( description='CLI tool to start a local instance of Pebble or Boulder CA server.') parser.add_argument('--server-type', '-s', @@ -236,9 +260,15 @@ def main(): help='specify the DNS server as `IP:PORT` to use by ' 'Pebble; if not specified, a local mock DNS server will be used to ' 'resolve domains to localhost.') + parser.add_argument('--http-01-port', type=int, default=DEFAULT_HTTP_01_PORT, + help='specify the port to use for http-01 validation; ' + 'this is currently only supported for Pebble.') args = parser.parse_args() - acme_server = ACMEServer(args.server_type, [], http_proxy=False, stdout=True, dns_server=args.dns_server) + acme_server = ACMEServer( + args.server_type, [], http_proxy=False, stdout=True, + dns_server=args.dns_server, http_01_port=args.http_01_port, + ) try: with acme_server as acme_xdist: diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py index a71c610e5..c9e46cdc7 100755 --- a/certbot-ci/certbot_integration_tests/utils/certbot_call.py +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -2,12 +2,13 @@ """Module to call certbot in test mode""" from __future__ import absolute_import -from distutils.version import LooseVersion import os import subprocess import sys +from distutils.version import LooseVersion import certbot_integration_tests +# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils.constants import * @@ -35,6 +36,8 @@ def certbot_test(certbot_args, directory_url, http_01_port, tls_alpn_01_port, def _prepare_environ(workspace): + # pylint: disable=missing-function-docstring + new_environ = os.environ.copy() new_environ['TMPDIR'] = workspace @@ -58,8 +61,13 @@ def _prepare_environ(workspace): # certbot_integration_tests.__file__ is: # '/path/to/certbot/certbot-ci/certbot_integration_tests/__init__.pyc' # ... and we want '/path/to/certbot' - certbot_root = os.path.dirname(os.path.dirname(os.path.dirname(certbot_integration_tests.__file__))) - python_paths = [path for path in new_environ['PYTHONPATH'].split(':') if path != certbot_root] + certbot_root = os.path.dirname(os.path.dirname( + os.path.dirname(certbot_integration_tests.__file__)) + ) + python_paths = [ + path for path in new_environ['PYTHONPATH'].split(':') + if path != certbot_root + ] new_environ['PYTHONPATH'] = ':'.join(python_paths) return new_environ @@ -70,7 +78,8 @@ def _compute_additional_args(workspace, environ, force_renew): output = subprocess.check_output(['certbot', '--version'], universal_newlines=True, stderr=subprocess.STDOUT, cwd=workspace, env=environ) - version_str = output.split(' ')[1].strip() # Typical response is: output = 'certbot 0.31.0.dev0' + # Typical response is: output = 'certbot 0.31.0.dev0' + version_str = output.split(' ')[1].strip() if LooseVersion(version_str) >= LooseVersion('0.30.0'): additional_args.append('--no-random-sleep-on-renew') @@ -92,6 +101,7 @@ def _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_por '--no-verify-ssl', '--http-01-port', str(http_01_port), '--https-port', str(tls_alpn_01_port), + '--manual-public-ip-logging-ok', '--config-dir', config_dir, '--work-dir', os.path.join(workspace, 'work'), '--logs-dir', os.path.join(workspace, 'logs'), @@ -112,11 +122,12 @@ def _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_por def main(): + # pylint: disable=missing-function-docstring args = sys.argv[1:] # Default config is pebble directory_url = os.environ.get('SERVER', PEBBLE_DIRECTORY_URL) - http_01_port = int(os.environ.get('HTTP_01_PORT', HTTP_01_PORT)) + http_01_port = int(os.environ.get('HTTP_01_PORT', DEFAULT_HTTP_01_PORT)) tls_alpn_01_port = int(os.environ.get('TLS_ALPN_01_PORT', TLS_ALPN_01_PORT)) # Execution of certbot in a self-contained workspace diff --git a/certbot-ci/certbot_integration_tests/utils/constants.py b/certbot-ci/certbot_integration_tests/utils/constants.py index 8b002478e..b02c434db 100644 --- a/certbot-ci/certbot_integration_tests/utils/constants.py +++ b/certbot-ci/certbot_integration_tests/utils/constants.py @@ -1,5 +1,5 @@ """Some useful constants to use throughout certbot-ci integration tests""" -HTTP_01_PORT = 5002 +DEFAULT_HTTP_01_PORT = 5002 TLS_ALPN_01_PORT = 5001 CHALLTESTSRV_PORT = 8055 BOULDER_V1_DIRECTORY_URL = 'http://localhost:4000/directory' @@ -7,4 +7,4 @@ BOULDER_V2_DIRECTORY_URL = 'http://localhost:4001/directory' PEBBLE_DIRECTORY_URL = 'https://localhost:14000/dir' PEBBLE_MANAGEMENT_URL = 'https://localhost:15000' MOCK_OCSP_SERVER_PORT = 4002 -PEBBLE_ALTERNATE_ROOTS = 2 \ No newline at end of file +PEBBLE_ALTERNATE_ROOTS = 2 diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index 779d736e3..416f6567e 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -4,7 +4,6 @@ from __future__ import print_function import os import os.path -from pkg_resources import resource_filename import shutil import socket import subprocess @@ -12,13 +11,14 @@ import sys import tempfile import time +from pkg_resources import resource_filename -BIND_DOCKER_IMAGE = 'internetsystemsconsortium/bind9:9.16' -BIND_BIND_ADDRESS = ('127.0.0.1', 45953) +BIND_DOCKER_IMAGE = "internetsystemsconsortium/bind9:9.16" +BIND_BIND_ADDRESS = ("127.0.0.1", 45953) # A TCP DNS message which is a query for '. CH A' transaction ID 0xcb37. This is used # by _wait_until_ready to check that BIND is responding without depending on dnspython. -BIND_TEST_QUERY = bytearray.fromhex('0011cb37000000010000000000000000010003') +BIND_TEST_QUERY = bytearray.fromhex("0011cb37000000010000000000000000010003") class DNSServer(object): @@ -31,7 +31,7 @@ class DNSServer(object): future to support parallelization (https://github.com/certbot/certbot/issues/8455). """ - def __init__(self, nodes, show_output=False): + def __init__(self, unused_nodes, show_output=False): """ Create an DNSServer instance. :param list nodes: list of node names that will be setup by pytest xdist @@ -40,16 +40,13 @@ class DNSServer(object): self.bind_root = tempfile.mkdtemp() - self.process = None + self.process = None # type: subprocess.Popen - self.dns_xdist = { - 'address': BIND_BIND_ADDRESS[0], - 'port': BIND_BIND_ADDRESS[1] - } + self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]} # Unfortunately the BIND9 image forces everything to stderr with -g and we can't # modify the verbosity. - self._output = sys.stderr if show_output else open(os.devnull, 'w') + self._output = sys.stderr if show_output else open(os.devnull, "w") def start(self): """Start the DNS server""" @@ -63,11 +60,11 @@ class DNSServer(object): def stop(self): """Stop the DNS server, and clean its resources""" if self.process: - try: - self.process.terminate() - self.process.wait() - except BaseException as e: - print("BIND9 did not stop cleanly: {}".format(e), file=sys.stderr) + try: + self.process.terminate() + self.process.wait() + except BaseException as e: + print("BIND9 did not stop cleanly: {}".format(e), file=sys.stderr) shutil.rmtree(self.bind_root, ignore_errors=True) @@ -76,65 +73,79 @@ class DNSServer(object): def _configure_bind(self): """Configure the BIND9 server based on the prebaked configuration""" - bind_conf_src = resource_filename('certbot_integration_tests', 'assets/bind-config') - for dir in ('conf', 'zones'): - shutil.copytree(os.path.join(bind_conf_src, dir), os.path.join(self.bind_root, dir)) + bind_conf_src = resource_filename( + "certbot_integration_tests", "assets/bind-config" + ) + for directory in ("conf", "zones"): + shutil.copytree( + os.path.join(bind_conf_src, directory), os.path.join(self.bind_root, directory) + ) def _start_bind(self): """Launch the BIND9 server as a Docker container""" - addr_str = '{}:{}'.format(BIND_BIND_ADDRESS[0], BIND_BIND_ADDRESS[1]) - self.process = subprocess.Popen([ - 'docker', 'run', '--rm', - '-p', '{}:53/udp'.format(addr_str), - '-p', '{}:53/tcp'.format(addr_str), - '-v', '{}/conf:/etc/bind'.format(self.bind_root), - '-v', '{}/zones:/var/lib/bind'.format(self.bind_root), - BIND_DOCKER_IMAGE - ], stdout=self._output, stderr=self._output) + addr_str = "{}:{}".format(BIND_BIND_ADDRESS[0], BIND_BIND_ADDRESS[1]) + self.process = subprocess.Popen( + [ + "docker", + "run", + "--rm", + "-p", + "{}:53/udp".format(addr_str), + "-p", + "{}:53/tcp".format(addr_str), + "-v", + "{}/conf:/etc/bind".format(self.bind_root), + "-v", + "{}/zones:/var/lib/bind".format(self.bind_root), + BIND_DOCKER_IMAGE, + ], + stdout=self._output, + stderr=self._output, + ) if self.process.poll(): - raise("BIND9 server stopped unexpectedly") + raise ValueError("BIND9 server stopped unexpectedly") try: - self._wait_until_ready() + self._wait_until_ready() except: - # The container might be running even if we think it isn't - self.stop() - raise + # The container might be running even if we think it isn't + self.stop() + raise def _wait_until_ready(self, attempts=30): - # type: (int) -> None - """ - Polls the DNS server over TCP until it gets a response, or until - it runs out of attempts and raises a ValueError. - The DNS response message must match the txn_id of the DNS query message, - but otherwise the contents are ignored. - :param int attempts: The number of attempts to make. - """ - for _ in range(attempts): - if self.process.poll(): - raise ValueError('BIND9 server stopped unexpectedly') + # type: (int) -> None + """ + Polls the DNS server over TCP until it gets a response, or until + it runs out of attempts and raises a ValueError. + The DNS response message must match the txn_id of the DNS query message, + but otherwise the contents are ignored. + :param int attempts: The number of attempts to make. + """ + for _ in range(attempts): + if self.process.poll(): + raise ValueError("BIND9 server stopped unexpectedly") - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(5.0) - try: - sock.connect(BIND_BIND_ADDRESS) - sock.sendall(BIND_TEST_QUERY) - buf = sock.recv(1024) - # We should receive a DNS message with the same tx_id - if buf and len(buf) > 4 and buf[2:4] == BIND_TEST_QUERY[2:4]: - return - # If we got a response but it wasn't the one we wanted, wait a little - time.sleep(1) - except: - # If there was a network error, wait a little - time.sleep(1) - pass - finally: - sock.close() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(5.0) + try: + sock.connect(BIND_BIND_ADDRESS) + sock.sendall(BIND_TEST_QUERY) + buf = sock.recv(1024) + # We should receive a DNS message with the same tx_id + if buf and len(buf) > 4 and buf[2:4] == BIND_TEST_QUERY[2:4]: + return + # If we got a response but it wasn't the one we wanted, wait a little + time.sleep(1) + except: # pylint: disable=bare-except + # If there was a network error, wait a little + time.sleep(1) + finally: + sock.close() - raise ValueError( - 'Gave up waiting for DNS server {} to respond'.format(BIND_BIND_ADDRESS)) + raise ValueError( + "Gave up waiting for DNS server {} to respond".format(BIND_BIND_ADDRESS) + ) def __enter__(self): self.start() diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index d83f276ef..799b079fe 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -39,6 +39,7 @@ def _suppress_x509_verification_warnings(): urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) except ImportError: # Handle old versions of request with vendorized urllib3 + # pylint: disable=no-member from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) @@ -256,7 +257,8 @@ def generate_csr(domains, key_path, csr_path, key_type=RSA_KEY_TYPE): def read_certificate(cert_path): """ - Load the certificate from the provided path, and return a human readable version of it (TEXT mode). + Load the certificate from the provided path, and return a human readable version + of it (TEXT mode). :param str cert_path: the path to the certificate :returns: the TEXT version of the certificate, as it would be displayed by openssl binary """ diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index 7fe03b990..cd62e1a7f 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -1,3 +1,5 @@ +# pylint: disable=missing-module-docstring + import json import os import stat @@ -5,18 +7,19 @@ import stat import pkg_resources import requests -from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT +from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT, MOCK_OCSP_SERVER_PORT PEBBLE_VERSION = 'v2.3.0' ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') -def fetch(workspace): +def fetch(workspace, http_01_port=DEFAULT_HTTP_01_PORT): + # pylint: disable=missing-function-docstring suffix = 'linux-amd64' if os.name != 'nt' else 'windows-amd64.exe' pebble_path = _fetch_asset('pebble', suffix) challtestsrv_path = _fetch_asset('pebble-challtestsrv', suffix) - pebble_config_path = _build_pebble_config(workspace) + pebble_config_path = _build_pebble_config(workspace, http_01_port) return pebble_path, challtestsrv_path, pebble_config_path @@ -35,7 +38,7 @@ def _fetch_asset(asset, suffix): return asset_path -def _build_pebble_config(workspace): +def _build_pebble_config(workspace, http_01_port): config_path = os.path.join(workspace, 'pebble-config.json') with open(config_path, 'w') as file_h: file_h.write(json.dumps({ @@ -44,7 +47,7 @@ def _build_pebble_config(workspace): 'managementListenAddress': '0.0.0.0:15000', 'certificate': os.path.join(ASSETS_PATH, 'cert.pem'), 'privateKey': os.path.join(ASSETS_PATH, 'key.pem'), - 'httpPort': 5002, + 'httpPort': http_01_port, 'tlsPort': 5001, 'ocspResponderURL': 'http://127.0.0.1:{0}'.format(MOCK_OCSP_SERVER_PORT), }, diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py index 9458560e8..b86e1cbc9 100755 --- a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py @@ -21,6 +21,7 @@ from certbot_integration_tests.utils.misc import GracefulTCPServer class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): + # pylint: disable=missing-function-docstring def do_POST(self): request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediate-keys/0', verify=False) issuer_key = serialization.load_pem_private_key(request.content, None, default_backend()) @@ -35,20 +36,28 @@ class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len)) response = requests.get('{0}/cert-status-by-serial/{1}'.format( - PEBBLE_MANAGEMENT_URL, str(hex(ocsp_request.serial_number)).replace('0x', '')), verify=False) + PEBBLE_MANAGEMENT_URL, str(hex(ocsp_request.serial_number)).replace('0x', '')), + verify=False + ) if not response.ok: - ocsp_response = ocsp.OCSPResponseBuilder.build_unsuccessful(ocsp.OCSPResponseStatus.UNAUTHORIZED) + ocsp_response = ocsp.OCSPResponseBuilder.build_unsuccessful( + ocsp.OCSPResponseStatus.UNAUTHORIZED + ) else: data = response.json() now = datetime.datetime.utcnow() cert = x509.load_pem_x509_certificate(data['Certificate'].encode(), default_backend()) if data['Status'] != 'Revoked': - ocsp_status, revocation_time, revocation_reason = ocsp.OCSPCertStatus.GOOD, None, None + ocsp_status = ocsp.OCSPCertStatus.GOOD + revocation_time = None + revocation_reason = None else: - ocsp_status, revocation_reason = ocsp.OCSPCertStatus.REVOKED, x509.ReasonFlags.unspecified - revoked_at = re.sub(r'( \+\d{4}).*$', r'\1', data['RevokedAt']) # "... +0000 UTC" => "+0000" + ocsp_status = ocsp.OCSPCertStatus.REVOKED + revocation_reason = x509.ReasonFlags.unspecified + # "... +0000 UTC" => "+0000" + revoked_at = re.sub(r'( \+\d{4}).*$', r'\1', data['RevokedAt']) revocation_time = parser.parse(revoked_at) ocsp_response = ocsp.OCSPResponseBuilder().add_response( diff --git a/certbot-ci/certbot_integration_tests/utils/proxy.py b/certbot-ci/certbot_integration_tests/utils/proxy.py index 3a16adebf..225f98e6e 100644 --- a/certbot-ci/certbot_integration_tests/utils/proxy.py +++ b/certbot-ci/certbot_integration_tests/utils/proxy.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +# pylint: disable=missing-module-docstring + import json import re import sys @@ -10,7 +12,9 @@ from certbot_integration_tests.utils.misc import GracefulTCPServer def _create_proxy(mapping): + # pylint: disable=missing-function-docstring class ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): + # pylint: disable=missing-class-docstring def do_GET(self): headers = {key.lower(): value for key, value in self.headers.items()} backend = [backend for pattern, backend in mapping.items() diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index ce29fe45d..4d4557939 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -18,7 +18,7 @@ install_requires = [ 'python-dateutil', 'pyyaml', 'requests', - 'six', + 'six' ] # Add pywin32 on Windows platforms to handle low-level system calls. diff --git a/certbot-ci/windows_installer_integration_tests/conftest.py b/certbot-ci/windows_installer_integration_tests/conftest.py index e36654f90..c6a89c323 100644 --- a/certbot-ci/windows_installer_integration_tests/conftest.py +++ b/certbot-ci/windows_installer_integration_tests/conftest.py @@ -9,8 +9,6 @@ See https://docs.pytest.org/en/latest/reference.html#hook-reference from __future__ import print_function import os -import pytest - ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 66e1c9f03..c894e5dee 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py index d59862a3c..81c053c04 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_cloudflare.dns_cloudflare` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Cloudflare API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 34938aca1..a00f06a8a 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py index 6ddbdfe5a..0d098445c 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_cloudxns.dns_cloudxns` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the CloudXNS API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 81f6afa12..3771c1d34 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py index 3ab8df041..2cb7a92de 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_digitalocean.dns_digitalocean` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the DigitalOcean API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index 75e25a848..e0c9561a2 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -19,7 +19,8 @@ class Authenticator(dns_common.DNSAuthenticator): This Authenticator uses the DigitalOcean API to fulfill a dns-01 challenge. """ - description = 'Obtain certs using a DNS TXT record (if you are using DigitalOcean for DNS).' + description = 'Obtain certificates using a DNS TXT record (if you are ' + \ + 'using DigitalOcean for DNS).' def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index f6c5936ae..f168ee06a 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py index f8a2e83aa..0f6168a13 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_dnsimple.dns_dnsimple` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the DNSimple API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index e0d95eea1..f23bd6668 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/__init__.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/__init__.py index 52f055237..fa49ee516 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/__init__.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_dnsmadeeasy.dns_dnsmadeeasy` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the DNS Made Easy API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 0ff3c71ab..e654ed421 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py b/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py index fdcb8cd48..fd81d0712 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_gehirn.dns_gehirn` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Gehirn Infrastructure Service DNS API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index e57796e1a..a856f1cde 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py index b88260b07..2d448c590 100644 --- a/certbot-dns-google/certbot_dns_google/__init__.py +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_google.dns_google` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Google Cloud DNS API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index 4eaed1783..4b0d91463 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -85,9 +85,13 @@ class _GoogleClient(object): scopes = ['https://www.googleapis.com/auth/ndev.clouddns.readwrite'] if account_json is not None: - credentials = ServiceAccountCredentials.from_json_keyfile_name(account_json, scopes) - with open(account_json) as account: - self.project_id = json.load(account)['project_id'] + try: + credentials = ServiceAccountCredentials.from_json_keyfile_name(account_json, scopes) + with open(account_json) as account: + self.project_id = json.load(account)['project_id'] + except Exception as e: + raise errors.PluginError( + "Error parsing credentials file '{}': {}".format(account_json, e)) else: credentials = None self.project_id = self.get_project_id() @@ -114,10 +118,13 @@ class _GoogleClient(object): record_contents = self.get_existing_txt_rrset(zone_id, record_name) if record_contents is None: - record_contents = [] - add_records = record_contents[:] + # If it wasn't possible to fetch the records at this label (missing .list permission), + # assume there aren't any (#5678). If there are actually records here, this will fail + # with HTTP 409/412 API errors. + record_contents = {"rrdatas": []} + add_records = record_contents["rrdatas"][:] - if "\""+record_content+"\"" in record_contents: + if "\""+record_content+"\"" in record_contents["rrdatas"]: # The process was interrupted previously and validation token exists return @@ -136,15 +143,15 @@ class _GoogleClient(object): ], } - if record_contents: + if record_contents["rrdatas"]: # We need to remove old records in the same request data["deletions"] = [ { "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": record_contents, - "ttl": record_ttl, + "rrdatas": record_contents["rrdatas"], + "ttl": record_contents["ttl"], }, ] @@ -184,7 +191,10 @@ class _GoogleClient(object): record_contents = self.get_existing_txt_rrset(zone_id, record_name) if record_contents is None: - record_contents = ["\"" + record_content + "\""] + # If it wasn't possible to fetch the records at this label (missing .list permission), + # assume there aren't any (#5678). If there are actually records here, this will fail + # with HTTP 409/412 API errors. + record_contents = {"rrdatas": ["\"" + record_content + "\""], "ttl": record_ttl} data = { "kind": "dns#change", @@ -193,14 +203,15 @@ class _GoogleClient(object): "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": record_contents, - "ttl": record_ttl, + "rrdatas": record_contents["rrdatas"], + "ttl": record_contents["ttl"], }, ], } # Remove the record being deleted from the list - readd_contents = [r for r in record_contents if r != "\"" + record_content + "\""] + readd_contents = [r for r in record_contents["rrdatas"] + if r != "\"" + record_content + "\""] if readd_contents: # We need to remove old records in the same request data["additions"] = [ @@ -209,7 +220,7 @@ class _GoogleClient(object): "type": "TXT", "name": record_name + ".", "rrdatas": readd_contents, - "ttl": record_ttl, + "ttl": record_contents["ttl"], }, ] @@ -231,14 +242,15 @@ class _GoogleClient(object): :param str zone_id: The ID of the managed zone. :param str record_name: The record name (typically beginning with '_acme-challenge.'). - :returns: List of TXT record values or None - :rtype: `list` of `string` or `None` + :returns: The resourceRecordSet corresponding to `record_name` or None + :rtype: `resourceRecordSet ` or `None` # pylint: disable=line-too-long """ rrs_request = self.dns.resourceRecordSets() - request = rrs_request.list(managedZone=zone_id, project=self.project_id) # Add dot as the API returns absolute domains record_name += "." + request = rrs_request.list(project=self.project_id, managedZone=zone_id, name=record_name, + type="TXT") try: response = request.execute() except googleapiclient_errors.Error: @@ -246,10 +258,8 @@ class _GoogleClient(object): "requesting a wildcard certificate, this might not work.") logger.debug("Error was:", exc_info=True) else: - if response: - for rr in response["rrsets"]: - if rr["name"] == record_name and rr["type"] == "TXT": - return rr["rrdatas"] + if response and response["rrsets"]: + return response["rrsets"][0] return None def _find_managed_zone_id(self, domain): diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index c52215f33..82c2a9102 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 5af027cef..396a6c8bd 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -70,7 +70,7 @@ class GoogleClientTest(unittest.TestCase): zone = "ZONE_ID" change = "an-id" - def _setUp_client_with_mock(self, zone_request_side_effect): + def _setUp_client_with_mock(self, zone_request_side_effect, rrs_list_side_effect=None): from certbot_dns_google._internal.dns_google import _GoogleClient pwd = os.path.dirname(__file__) @@ -86,9 +86,16 @@ class GoogleClientTest(unittest.TestCase): mock_mz.list.return_value.execute.side_effect = zone_request_side_effect mock_rrs = mock.MagicMock() - rrsets = {"rrsets": [{"name": "_acme-challenge.example.org.", "type": "TXT", - "rrdatas": ["\"example-txt-contents\""]}]} - mock_rrs.list.return_value.execute.return_value = rrsets + def rrs_list(project=None, managedZone=None, name=None, type=None): + response = {"rrsets": []} + if name == "_acme-challenge.example.org.": + response = {"rrsets": [{"name": "_acme-challenge.example.org.", "type": "TXT", + "rrdatas": ["\"example-txt-contents\""], "ttl": 60}]} + mock_return = mock.MagicMock() + mock_return.execute.return_value = response + mock_return.execute.side_effect = rrs_list_side_effect + return mock_return + mock_rrs.list.side_effect = rrs_list mock_changes = mock.MagicMock() client.dns.managedZones = mock.MagicMock(return_value=mock_mz) @@ -107,6 +114,17 @@ class GoogleClientTest(unittest.TestCase): self.assertFalse(credential_mock.called) self.assertTrue(get_project_id_mock.called) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + def test_client_bad_credentials_file(self, credential_mock): + credential_mock.side_effect = ValueError('Some exception buried in oauth2client') + with self.assertRaises(errors.PluginError) as cm: + self._setUp_client_with_mock([]) + self.assertEqual( + str(cm.exception), + "Error parsing credentials file '/not/a/real/path.json': " + "Some exception buried in oauth2client" + ) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) @@ -162,11 +180,29 @@ class GoogleClientTest(unittest.TestCase): # pylint: disable=line-too-long mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" with mock.patch(mock_get_rrs) as mock_rrs: - mock_rrs.return_value = ["sample-txt-contents"] + mock_rrs.return_value = {"rrdatas": ["sample-txt-contents"], "ttl": self.record_ttl} client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) self.assertTrue(changes.create.called) - self.assertTrue("sample-txt-contents" in - changes.create.call_args_list[0][1]["body"]["deletions"][0]["rrdatas"]) + deletions = changes.create.call_args_list[0][1]["body"]["deletions"][0] + self.assertTrue("sample-txt-contents" in deletions["rrdatas"]) + self.assertEqual(self.record_ttl, deletions["ttl"]) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google._internal.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_delete_old_ttl_case(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + # pylint: disable=line-too-long + mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + custom_ttl = 300 + mock_rrs.return_value = {"rrdatas": ["sample-txt-contents"], "ttl": custom_ttl} + client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + self.assertTrue(changes.create.called) + deletions = changes.create.call_args_list[0][1]["body"]["deletions"][0] + self.assertTrue("sample-txt-contents" in deletions["rrdatas"]) + self.assertEqual(custom_ttl, deletions["ttl"]) #otherwise HTTP 412 @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', @@ -210,14 +246,13 @@ class GoogleClientTest(unittest.TestCase): @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) - def test_del_txt_record(self, unused_credential_mock): + def test_del_txt_record_multi_rrdatas(self, unused_credential_mock): client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) - # pylint: disable=line-too-long mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" with mock.patch(mock_get_rrs) as mock_rrs: - mock_rrs.return_value = ["\"sample-txt-contents\"", - "\"example-txt-contents\""] + mock_rrs.return_value = {"rrdatas": ["\"sample-txt-contents\"", + "\"example-txt-contents\""], "ttl": self.record_ttl} client.del_txt_record(DOMAIN, "_acme-challenge.example.org", "example-txt-contents", self.record_ttl) @@ -250,19 +285,48 @@ class GoogleClientTest(unittest.TestCase): @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) - def test_del_txt_record_error_during_zone_lookup(self, unused_credential_mock): - client, unused_changes = self._setUp_client_with_mock(API_ERROR) + def test_del_txt_record_single_rrdatas(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + # pylint: disable=line-too-long + mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + mock_rrs.return_value = {"rrdatas": ["\"example-txt-contents\""], "ttl": self.record_ttl} + client.del_txt_record(DOMAIN, "_acme-challenge.example.org", + "example-txt-contents", self.record_ttl) + expected_body = { + "kind": "dns#change", + "deletions": [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": "_acme-challenge.example.org.", + "rrdatas": ["\"example-txt-contents\""], + "ttl": self.record_ttl, + }, + ], + } + + changes.create.assert_called_with(body=expected_body, + managedZone=self.zone, + project=PROJECT_ID) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google._internal.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_del_txt_record_error_during_zone_lookup(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock(API_ERROR) client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + changes.create.assert_not_called() @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) def test_del_txt_record_zone_not_found(self, unused_credential_mock): - client, unused_changes = self._setUp_client_with_mock([{'managedZones': []}, + client, changes = self._setUp_client_with_mock([{'managedZones': []}, {'managedZones': []}]) - client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + changes.create.assert_not_called() @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', @@ -276,24 +340,39 @@ class GoogleClientTest(unittest.TestCase): @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) - def test_get_existing(self, unused_credential_mock): + def test_get_existing_found(self, unused_credential_mock): client, unused_changes = self._setUp_client_with_mock( [{'managedZones': [{'id': self.zone}]}]) # Record name mocked in setUp found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") - self.assertEqual(found, ["\"example-txt-contents\""]) + self.assertEqual(found["rrdatas"], ["\"example-txt-contents\""]) + self.assertEqual(found["ttl"], 60) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google._internal.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing_not_found(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") self.assertEqual(not_found, None) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google._internal.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing_with_error(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}], API_ERROR) + # Record name mocked in setUp + found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") + self.assertEqual(found, None) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) def test_get_existing_fallback(self, unused_credential_mock): client, unused_changes = self._setUp_client_with_mock( - [{'managedZones': [{'id': self.zone}]}]) - mock_execute = client.dns.resourceRecordSets.return_value.list.return_value.execute - mock_execute.side_effect = API_ERROR - + [{'managedZones': [{'id': self.zone}]}], API_ERROR) rrset = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") self.assertFalse(rrset) diff --git a/certbot-dns-linode/certbot_dns_linode/__init__.py b/certbot-dns-linode/certbot_dns_linode/__init__.py index 4bfd95573..bca15bdb2 100644 --- a/certbot-dns-linode/certbot_dns_linode/__init__.py +++ b/certbot-dns-linode/certbot_dns_linode/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_linode.dns_linode` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Linode API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py index f9450c02c..c1b5e066f 100644 --- a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py @@ -24,7 +24,7 @@ class Authenticator(dns_common.DNSAuthenticator): This Authenticator uses the Linode API to fulfill a dns-01 challenge. """ - description = 'Obtain certs using a DNS TXT record (if you are using Linode for DNS).' + description = 'Obtain certificates using a DNS TXT record (if you are using Linode for DNS).' def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index e7cdcabfb..a6f159757 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/certbot_dns_luadns/__init__.py b/certbot-dns-luadns/certbot_dns_luadns/__init__.py index e8e86f77c..302cb1392 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/__init__.py +++ b/certbot-dns-luadns/certbot_dns_luadns/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_luadns.dns_luadns` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the LuaDNS API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index d38c9fcca..ff4a1b41d 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/certbot_dns_nsone/__init__.py b/certbot-dns-nsone/certbot_dns_nsone/__init__.py index e59be74a7..6c7d41ba4 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/__init__.py +++ b/certbot-dns-nsone/certbot_dns_nsone/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_nsone.dns_nsone` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the NS1 API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index deaa60790..887d5120a 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/certbot_dns_ovh/__init__.py b/certbot-dns-ovh/certbot_dns_ovh/__init__.py index d508fad1b..6a079e59f 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/__init__.py +++ b/certbot-dns-ovh/certbot_dns_ovh/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_ovh.dns_ovh` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the OVH API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 349903900..d519a9e18 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py index da8ef3419..3c574835f 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_rfc2136.dns_rfc2136` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using RFC 2136 Dynamic Updates. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index c2c97a41f..540fc1a67 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/certbot_dns_route53/__init__.py b/certbot-dns-route53/certbot_dns_route53/__init__.py index 8659617ef..1b59f5620 100644 --- a/certbot-dns-route53/certbot_dns_route53/__init__.py +++ b/certbot-dns-route53/certbot_dns_route53/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_route53.dns_route53` plugin automates the process of completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Amazon Web Services Route 53 API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 7d28d2703..cffa16367 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py index f18780c18..c16ee96ef 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/__init__.py @@ -3,6 +3,10 @@ The `~certbot_dns_sakuracloud.dns_sakuracloud` plugin automates the process of c a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently removing, TXT records using the Sakura Cloud DNS API. +.. note:: + The plugin is not installed by default. It can be installed by heading to + `certbot.eff.org `_, choosing your system and + selecting the Wildcard tab. Named Arguments --------------- diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 22f5fa031..2c88f1226 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 87afedd38..15fbe61f7 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -226,7 +226,7 @@ class NginxConfigurator(common.Installer): if not fullchain_path: raise errors.PluginError( "The nginx plugin currently requires --fullchain-path to " - "install a cert.") + "install a certificate.") vhosts = self.choose_vhosts(domain, create_if_no_match=True) for vhost in vhosts: diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index 2f6458f87..40f994988 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -1,5 +1,6 @@ """A class that performs HTTP-01 challenges for Nginx""" +import io import logging from acme import challenges @@ -102,7 +103,7 @@ class NginxHttp01(common.ChallengePerformer): self.configurator.reverter.register_file_creation( True, self.challenge_conf) - with open(self.challenge_conf, "w") as new_conf: + with io.open(self.challenge_conf, "w", encoding="utf-8") as new_conf: nginxparser.dump(config, new_conf) def _default_listen_addresses(self): diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index a8ac90427..5f723dcef 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -16,6 +16,7 @@ from pyparsing import stringEnd from pyparsing import White from pyparsing import ZeroOrMore import six +from acme.magic_typing import IO, Any # pylint: disable=unused-import logger = logging.getLogger(__name__) @@ -130,26 +131,27 @@ def load(_file): def dumps(blocks): - """Dump to a string. + # type: (UnspacedList) -> six.text_type + """Dump to a Unicode string. :param UnspacedList block: The parsed tree - :param int indentation: The number of spaces to indent - :rtype: str + :rtype: six.text_type """ - return str(RawNginxDumper(blocks.spaced)) + return six.text_type(RawNginxDumper(blocks.spaced)) def dump(blocks, _file): + # type: (UnspacedList, IO[Any]) -> None """Dump to a file. :param UnspacedList block: The parsed tree - :param file _file: The file to dump to - :param int indentation: The number of spaces to indent - :rtype: NoneType + :param IO[Any] _file: The file stream to dump to. It must be opened with + Unicode encoding. + :rtype: None """ - return _file.write(dumps(blocks)) + _file.write(dumps(blocks)) spacey = lambda x: (isinstance(x, six.string_types) and x.isspace()) or x == '' diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 641ffb020..72091b03f 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -249,7 +249,7 @@ class NginxParser(object): continue out = nginxparser.dumps(tree) logger.debug('Writing nginx conf tree to %s:\n%s', filename, out) - with open(filename, 'w') as _file: + with io.open(filename, 'w', encoding='utf-8') as _file: _file.write(out) except IOError: diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 9dfa7bbb3..0ed164da2 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.10.0.dev0' +version = '1.11.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 620d1b6de..0083c2448 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -492,6 +492,14 @@ class NginxParserTest(util.NginxTest): self.assertEqual(['server'], parsed[0][2][0]) self.assertEqual(['listen', '80'], parsed[0][2][1][3]) + def test_valid_unicode_roundtrip(self): + """This tests the parser's ability to load and save a config containing Unicode""" + nparser = parser.NginxParser(self.config_path) + nparser._parse_files( + nparser.abs_path('valid_unicode_comments.conf') + ) # pylint: disable=protected-access + nparser.filedump(lazy=False) + def test_invalid_unicode_characters(self): with self.assertLogs() as log: nparser = parser.NginxParser(self.config_path) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index a7f48ea3b..4c8de11af 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,42 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.10.0 - master +## 1.11.0 - master + +### Added + +* + +### Changed + +* We deprecated support for Python 2 in Certbot and its ACME library. + Support for Python 2 will be removed in the next planned release of Certbot. +* certbot-auto was deprecated on all systems. For more information about this + change, see + https://community.letsencrypt.org/t/certbot-auto-no-longer-works-on-debian-based-systems/139702/7. +* We deprecated support for Apache 2.2 in the certbot-apache plugin and it will + be removed in a future release of Certbot. + +### Fixed + +* The Certbot snap no longer loads packages installed via `pip install --user`. This + was unintended and DNS plugins should be installed via `snap` instead. +* `certbot-dns-google` would sometimes crash with HTTP 409/412 errors when used with very large zones. See [#6036](https://github.com/certbot/certbot/issues/6036). +* `certbot-dns-google` would sometimes crash with an HTTP 412 error if preexisting records had an unexpected TTL, i.e.: different than Certbot's default TTL for this plugin. See [#8551](https://github.com/certbot/certbot/issues/8551). + +More details about these changes can be found on our GitHub repo. + +## 1.10.1 - 2020-12-03 + +### Fixed + +* Fixed a bug in `certbot.util.add_deprecated_argument` that caused the + deprecated `--manual-public-ip-logging-ok` flag to crash Certbot in some + scenarios. + +More details about these changes can be found on our GitHub repo. + +## 1.10.0 - 2020-12-01 ### Added @@ -10,7 +45,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Confirmation when deleting certificates * CLI flag `--key-type` has been added to specify 'rsa' or 'ecdsa' (default 'rsa'). * CLI flag `--elliptic-curve` has been added which takes an NIST/SECG elliptic curve. Any of - `secp256r1`, `secp284r1` and `secp521r1` are accepted values. + `secp256r1`, `secp384r1` and `secp521r1` are accepted values. * The command `certbot certficates` lists the which type of the private key that was used for the private key. * Support for Python 3.9 was added to Certbot and all of its components. @@ -20,11 +55,10 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * certbot-auto was deprecated on Debian based systems. * CLI flag `--manual-public-ip-logging-ok` is now a no-op, generates a deprecation warning, and will be removed in a future release. -* ### Fixed -* +* Fixed a Unicode-related crash in the nginx plugin when running under Python 2. More details about these changes can be found on our GitHub repo. diff --git a/certbot/README.rst b/certbot/README.rst index 3bd5e4cd7..40f6a52ec 100644 --- a/certbot/README.rst +++ b/certbot/README.rst @@ -18,10 +18,6 @@ systems. To see the changes made to Certbot between versions please refer to our `changelog `_. -Until May 2016, Certbot was named simply ``letsencrypt`` or ``letsencrypt-auto``, -depending on install method. Instructions on the Internet, and some pieces of the -software, may still refer to this older name. - Contributing ------------ @@ -96,7 +92,7 @@ Current Features - apache/2.x - nginx/0.8.48+ - webroot (adds files to webroot directories in order to prove control of - domains and obtain certs) + domains and obtain certificates) - standalone (runs its own simple webserver to prove you control a domain) - other server software via `third party plugins `_ @@ -106,7 +102,7 @@ Current Features * Can get domain-validated (DV) certificates. * Can revoke certificates. * Adjustable RSA key bit-length (2048 (default), 4096, ...). -* Adjustable [EC](https://en.wikipedia.org/wiki/Elliptic-curve_cryptography) +* Adjustable `EC `_ key (`secp256r1` (default), `secp384r1`, `secp521r1`). * Can optionally install a http -> https redirect, so your site effectively runs https only (Apache only) diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 8ee415682..98009a71b 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,4 +1,13 @@ """Certbot client.""" +import warnings +import sys # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.10.0.dev0' +__version__ = '1.11.0.dev0' + +if sys.version_info[0] == 2: + warnings.warn( + "Python 2 support will be dropped in the next release of Certbot. " + "Please upgrade your Python version.", + PendingDeprecationWarning, + ) # pragma: no cover diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index 80a98ab04..dfbe4b538 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -369,7 +369,7 @@ def _describe_certs(config, parsed_certs, parse_failures): notify = out.append if not parsed_certs and not parse_failures: - notify("No certs found.") + notify("No certificates found.") else: if parsed_certs: match = "matching " if config.certname or config.domains else "" diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index 141dd41a4..2cb506157 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -2,8 +2,10 @@ from __future__ import print_function import argparse import copy +import functools import glob import sys + import configargparse import six import zope.component @@ -356,6 +358,18 @@ class HelpfulArgumentParser(object): :param dict **kwargs: various argparse settings for this argument """ + action = kwargs.get("action") + if action is util.DeprecatedArgumentAction: + # If the argument is deprecated through + # certbot.util.add_deprecated_argument, it is not shown in the help + # output and any value given to the argument is thrown away during + # argument parsing. Because of this, we handle this case early + # skipping putting the argument in different help topics and + # handling default detection since these actions aren't needed and + # can cause bugs like + # https://github.com/certbot/certbot/issues/8495. + self.parser.add_argument(*args, **kwargs) + return if isinstance(topics, list): # if this flag can be listed in multiple sections, try to pick the one @@ -410,8 +424,22 @@ class HelpfulArgumentParser(object): :param int nargs: Number of arguments the option takes. """ - util.add_deprecated_argument( - self.parser.add_argument, argument_name, num_args) + # certbot.util.add_deprecated_argument expects the normal add_argument + # interface provided by argparse. This is what is given including when + # certbot.util.add_deprecated_argument is used by plugins, however, in + # that case the first argument to certbot.util.add_deprecated_argument + # is certbot._internal.cli.HelpfulArgumentGroup.add_argument which + # internally calls the add method of this class. + # + # The difference between the add method of this class and the standard + # argparse add_argument method caused a bug in the past (see + # https://github.com/certbot/certbot/issues/8495) so we use the same + # code path here for consistency and to ensure it works. To do that, we + # wrap the add method in a similar way to + # HelpfulArgumentGroup.add_argument by providing a help topic (which in + # this case is set to None). + add_func = functools.partial(self.add, None) + util.add_deprecated_argument(add_func, argument_name, num_args) def add_group(self, topic, verbs=(), **kwargs): """Create a new argument group. diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 1d68bde59..d2286bd7a 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -5,6 +5,7 @@ from __future__ import print_function import functools import logging.handlers import sys +import warnings import configobj import josepy as jose @@ -253,7 +254,7 @@ def _handle_identical_cert_request(config, # type: configuration.NamespaceConfi elif config.verb == "certonly": keep_opt = "Keep the existing certificate for now" choices = [keep_opt, - "Renew & replace the cert (may be subject to CA rate limits)"] + "Renew & replace the certificate (may be subject to CA rate limits)"] display = zope.component.getUtility(interfaces.IDisplay) response = display.menu(question, choices, @@ -433,8 +434,8 @@ def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): _format_list("-", removed), br=os.linesep)) obj = zope.component.getUtility(interfaces.IDisplay) - if not obj.yesno(msg, "Update cert", "Cancel", default=True): - raise errors.ConfigurationError("Specified mismatched cert name and domains.") + if not obj.yesno(msg, "Update certificate", "Cancel", default=True): + raise errors.ConfigurationError("Specified mismatched certificate name and domains.") def _find_domains_or_certname(config, installer, question=None): @@ -512,7 +513,7 @@ def _report_new_cert(config, cert_path, fullchain_path, key_path=None): # and say something more informative here. msg = ('Congratulations! Your certificate and chain have been saved at:{br}' '{0}{br}{1}' - 'Your cert will expire on {2}. To obtain a new or tweaked version of this ' + 'Your certificate will expire on {2}. To obtain a new or tweaked version of this ' 'certificate in the future, simply run {3} again{4}. ' 'To non-interactively renew *all* of your certificates, run "{3} renew"' .format(fullchain_path, privkey_statement, expiry, cli.cli_command, verbswitch, @@ -596,8 +597,8 @@ def _delete_if_appropriate(config): attempt_deletion = config.delete_after_revoke if attempt_deletion is None: - msg = ("Would you like to delete the cert(s) you just revoked, along with all earlier and " - "later versions of the cert?") + msg = ("Would you like to delete the certificate(s) you just revoked, " + "along with all earlier and later versions of the certificate?") attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", force_interactive=True, default=True) @@ -619,8 +620,8 @@ def _delete_if_appropriate(config): cert_manager.match_and_check_overlaps(config, [lambda x: archive_dir], lambda x: x.archive_dir, lambda x: x) except errors.OverlappingMatchFound: - logger.warning("Not deleting revoked certs due to overlapping archive dirs. More than " - "one certificate is using %s", archive_dir) + logger.warning("Not deleting revoked certificates due to overlapping archive dirs. " + "More than one certificate is using %s", archive_dir) return except Exception as e: msg = ('config.default_archive_dir: {0}, config.live_dir: {1}, archive_dir: {2},' @@ -768,7 +769,7 @@ def update_account(config, unused_plugins): acc.regr = acc.regr.update(uri=prev_regr_uri) account_storage.update_regr(acc, cb_client.acme) - if config.email is None: + if not config.email: display_util.notify("Any contact information associated " "with this account has been removed.") else: @@ -1097,7 +1098,7 @@ def revoke(config, unused_plugins): raise errors.Error("Error! Exactly one of --cert-path or --cert-name must be specified!") if config.key_path is not None: # revocation by cert key - logger.debug("Revoking %s using cert key %s", + logger.debug("Revoking %s using certificate key %s", config.cert_path[0], config.key_path[0]) crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) key = jose.JWK.load(config.key_path[1]) @@ -1381,6 +1382,7 @@ def main(cli_args=None): plugins = plugins_disco.PluginsRegistry.find_all() logger.debug("certbot version: %s", certbot.__version__) + logger.debug("Location of certbot entry point: %s", sys.argv[0]) # do not log `config`, as it contains sensitive data (e.g. revoke --key)! logger.debug("Arguments: %r", cli_args) logger.debug("Discovered plugins: %r", plugins) @@ -1402,6 +1404,13 @@ def main(cli_args=None): if config.func != plugins_cmd: # pylint: disable=comparison-with-callable raise + if sys.version_info[0] == 2: + warnings.warn( + "Python 2 support will be dropped in the next release of Certbot. " + "Please upgrade your Python version.", + PendingDeprecationWarning, + ) # pragma: no cover + set_displayer(config) # Reporter diff --git a/certbot/certbot/_internal/plugins/null.py b/certbot/certbot/_internal/plugins/null.py index cf7c05a2b..5ab2f4a04 100644 --- a/certbot/certbot/_internal/plugins/null.py +++ b/certbot/certbot/_internal/plugins/null.py @@ -1,7 +1,6 @@ """Null plugin.""" import logging -import zope.component import zope.interface from certbot import interfaces diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index 1d008193d..3a550d355 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -19,6 +19,7 @@ import zope.component from acme.magic_typing import List from acme.magic_typing import Optional # pylint: disable=unused-import from certbot import crypto_util +from certbot.display import util as display_util from certbot import errors from certbot import interfaces from certbot import util @@ -98,7 +99,7 @@ def _reconstitute(config, full_path): config.domains = [util.enforce_domain_sanity(d) for d in renewal_candidate.names()] except errors.ConfigurationError as error: - logger.warning("Renewal configuration file %s references a cert " + logger.warning("Renewal configuration file %s references a certificate " "that contains an invalid domain name. The problem " "was: %s. Skipping.", full_path, error) return None @@ -292,13 +293,13 @@ def should_renew(config, lineage): def _avoid_invalidating_lineage(config, lineage, original_server): "Do not renew a valid cert with one from a staging server!" - # Some lineages may have begun with --staging, but then had production certs - # added to them + # Some lineages may have begun with --staging, but then had production + # certificates added to them with open(lineage.cert) as the_file: contents = the_file.read() latest_cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, contents) - # all our test certs are from happy hacker fake CA, though maybe one day + # all our test certificates are from happy hacker fake CA, though maybe one day # we should test more methodically now_valid = "fake" not in repr(latest_cert.get_issuer()).lower() @@ -347,40 +348,42 @@ def report(msgs, category): def _renew_describe_results(config, renew_successes, renew_failures, renew_skipped, parse_failures): + # type: (interfaces.IConfig, List[str], List[str], List[str], List[str]) -> None + """ + Print a report to the terminal about the results of the renewal process. - out = [] # type: List[str] - notify = out.append - disp = zope.component.getUtility(interfaces.IDisplay) + :param interfaces.IConfig config: Configuration + :param list renew_successes: list of fullchain paths which were renewed + :param list renew_failures: list of fullchain paths which failed to be renewed + :param list renew_skipped: list of messages to print about skipped certificates + :param list parse_failures: list of renewal parameter paths which had erorrs + """ + notify = display_util.notify + notify_error = logger.error - def notify_error(err): - """Notify and log errors.""" - notify(str(err)) - logger.error(err) + notify('\n{}'.format(display_util.SIDE_FRAME)) + + renewal_noun = "simulated renewal" if config.dry_run else "renewal" - if config.dry_run: - notify("** DRY RUN: simulating 'certbot renew' close to cert expiry") - notify("** (The test certificates below have not been saved.)") - notify("") if renew_skipped: - notify("The following certs are not due for renewal yet:") + notify("The following certificates are not due for renewal yet:") notify(report(renew_skipped, "skipped")) if not renew_successes and not renew_failures: - notify("No renewals were attempted.") + notify("No {renewal}s were attempted.".format(renewal=renewal_noun)) if (config.pre_hook is not None or config.renew_hook is not None or config.post_hook is not None): notify("No hooks were run.") elif renew_successes and not renew_failures: - notify("Congratulations, all renewals succeeded. The following certs " - "have been renewed:") + notify("Congratulations, all {renewal}s succeeded: ".format(renewal=renewal_noun)) notify(report(renew_successes, "success")) elif renew_failures and not renew_successes: - notify_error("All renewal attempts failed. The following certs could " - "not be renewed:") + notify_error("All %ss failed. The following certificates could " + "not be renewed:", renewal_noun) notify_error(report(renew_failures, "failure")) elif renew_failures and renew_successes: - notify("The following certs were successfully renewed:") + notify("The following {renewal}s succeeded:".format(renewal=renewal_noun)) notify(report(renew_successes, "success") + "\n") - notify_error("The following certs could not be renewed:") + notify_error("The following %ss failed:", renewal_noun) notify_error(report(renew_failures, "failure")) if parse_failures: @@ -388,11 +391,7 @@ def _renew_describe_results(config, renew_successes, renew_failures, "were invalid: ") notify(report(parse_failures, "parsefail")) - if config.dry_run: - notify("** DRY RUN: simulating 'certbot renew' close to cert expiry") - notify("** (The test certificates above have not been saved.)") - - disp.notification("\n".join(out), wrap=False) + notify(display_util.SIDE_FRAME) def handle_renewal_request(config): @@ -482,9 +481,10 @@ def handle_renewal_request(config): except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. - logger.warning("Attempting to renew cert (%s) from %s produced an " - "unexpected error: %s. Skipping.", lineagename, - renewal_file, e) + logger.error( + "Failed to renew certificate %s with error: %s", + lineagename, e + ) logger.debug("Traceback was:\n%s", traceback.format_exc()) renew_failures.append(renewal_candidate.fullchain) diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 7919a168f..ff58313e5 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -810,8 +810,8 @@ class RenewableCert(interfaces.RenewableCert): May need to recover from rare interrupted / crashed states.""" if self.has_pending_deployment(): - logger.warning("Found a new cert /archive/ that was not linked to in /live/; " - "fixing...") + logger.warning("Found a new certificate /archive/ that was not " + "linked to in /live/; fixing...") self.update_all_links_to(self.latest_common_version()) return False return True @@ -884,7 +884,7 @@ class RenewableCert(interfaces.RenewableCert): """ target = self.current_target("cert") if target is None: - raise errors.CertStorageError("could not find cert file") + raise errors.CertStorageError("could not find the certificate file") with open(target) as f: return crypto_util.get_names_from_cert(f.read()) diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index c8382402a..e0f85c1cd 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -279,7 +279,7 @@ def verify_renewable_cert_sig(renewable_cert): verify_signed_payload(pk, cert.signature, cert.tbs_certificate_bytes, cert.signature_hash_algorithm) except (IOError, ValueError, InvalidSignature) as e: - error_str = "verifying the signature of the cert located at {0} has failed. \ + error_str = "verifying the signature of the certificate located at {0} has failed. \ Details: {1}".format(renewable_cert.cert_path, e) logger.exception(error_str) raise errors.Error(error_str) @@ -330,7 +330,7 @@ def verify_cert_matches_priv_key(cert_path, key_path): context.use_privatekey_file(key_path) context.check_privatekey() except (IOError, SSL.Error) as e: - error_str = "verifying the cert located at {0} matches the \ + error_str = "verifying the certificate located at {0} matches the \ private key located at {1} has failed. \ Details: {2}".format(cert_path, key_path, e) diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 75ce9e2ff..b63338e2e 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -167,7 +167,7 @@ def _determine_ocsp_server(cert_path): if host: return url, host - logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) + logger.info("Cannot process OCSP host from URL (%s) in certificate at %s", url, cert_path) return None, None @@ -222,7 +222,7 @@ def _check_ocsp_cryptography(cert_path, chain_path, url, timeout): def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert, cert_path): - """Verify that the OCSP is valid for serveral criteria""" + """Verify that the OCSP is valid for several criteria""" # Assert OCSP response corresponds to the certificate we are talking about if response_ocsp.serial_number != request_ocsp.serial_number: raise AssertionError('the certificate in response does not correspond ' diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index fd7b46f85..8db5ab34a 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -439,7 +439,7 @@ def safe_email(email): return False -class _ShowWarning(argparse.Action): +class DeprecatedArgumentAction(argparse.Action): """Action to log a warning when an argument is used.""" def __call__(self, unused1, unused2, unused3, option_string=None): logger.warning("Use of %s is deprecated.", option_string) @@ -458,16 +458,16 @@ def add_deprecated_argument(add_argument, argument_name, nargs): :param nargs: Value for nargs when adding the argument to argparse. """ - if _ShowWarning not in configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE: + if DeprecatedArgumentAction not in configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE: # In version 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE was # changed from a set to a tuple. if isinstance(configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE, set): configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE.add( - _ShowWarning) + DeprecatedArgumentAction) else: configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE += ( - _ShowWarning,) - add_argument(argument_name, action=_ShowWarning, + DeprecatedArgumentAction,) + add_argument(argument_name, action=DeprecatedArgumentAction, help=argparse.SUPPRESS, nargs=nargs) diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index 5f8e2e00b..a320b30e8 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -1,4 +1,4 @@ -usage: +usage: certbot [SUBCOMMAND] [options] [-d DOMAIN] [-d DOMAIN] ... Certbot can obtain and install HTTPS/TLS/SSL certificates. By default, @@ -118,12 +118,12 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.9.0 (certbot(-auto); - OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). - The flags encoded in the user agent are: --duplicate, - --force-renew, --allow-subset-of-names, -n, and - whether any hooks are set. + "". (default: CertbotACMEClient/1.10.1 + (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX + Installer/YYY (SUBCOMMAND; flags: FLAGS) + Py/major.minor.patchlevel). The flags encoded in the + user agent are: --duplicate, --force-renew, --allow- + subset-of-names, -n, and whether any hooks are set. --user-agent-comment USER_AGENT_COMMENT Add a comment to the default user agent string. May be used when repackaging Certbot or calling it from @@ -188,12 +188,12 @@ security: Security parameters & server settings --rsa-key-size N Size of the RSA key. (default: 2048) - --key-type type The type of algorithm to use for the the private key. - Either ``rsa`` or ``ecdsa``. (default: ``rsa``). - --elliptic-curve The elliptic curve to use when choosing ``ecdsa`` as the key - type. Accepted values are SECG curve names as defined by - the cryptography library. ``secp256r1``, ``secp384r1``, - ``secp521r1``. (default: secp256r1). + --key-type {rsa,ecdsa} + Type of generated private key(Only *ONE* per + invocation can be provided at this time) (default: + rsa) + --elliptic-curve N The SECG elliptic curve name to use. Please see RFC + 8446 for supported values. (default: secp256r1) --must-staple Adds the OCSP Must Staple extension to the certificate. Autoconfigures OCSP Stapling for supported setups (Apache version >= 2.3.3 ). (default: @@ -694,8 +694,6 @@ manual: --manual-cleanup-hook MANUAL_CLEANUP_HOOK Path or command to execute for the cleanup script (default: None) - --manual-public-ip-logging-ok - Automatically allows public IP logging (default: Ask) nginx: Nginx Web Server plugin diff --git a/certbot/docs/compatibility.rst b/certbot/docs/compatibility.rst index a511f36a2..a4f33c281 100644 --- a/certbot/docs/compatibility.rst +++ b/certbot/docs/compatibility.rst @@ -9,7 +9,7 @@ application itself. This means that we will not change behavior in a backwards incompatible way except in a new major version of the project. .. note:: None of this applies to the behavior of Certbot distribution - mechanisms such as :ref:`certbot-auto ` or OS packages whose + mechanisms such as :ref:`our snaps ` or OS packages whose behavior may change at any time. Semantic versioning only applies to the common Certbot components that are installed by various distribution methods. diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index eb90b05f4..e130f0548 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -282,8 +282,8 @@ support for IIS, Icecast and Plesk. Installers and Authenticators will oftentimes be the same class/object (because for instance both tasks can be performed by a webserver like nginx) though this is not always the case (the standalone plugin is an authenticator -that listens on port 80, but it cannot install certs; a postfix plugin would -be an installer but not an authenticator). +that listens on port 80, but it cannot install certificates; a postfix plugin +would be an installer but not an authenticator). Installers and Authenticators are kept separate because it should be possible to use the `~.StandaloneAuthenticator` (it sets @@ -516,11 +516,13 @@ Steps: 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite including coverage. The ``--skip-missing-interpreters`` argument ignores missing versions of Python needed for running the tests. Fix any errors. -5. Submit the PR. Once your PR is open, please do not force push to the branch +5. If any documentation should be added or updated as part of the changes you + have made, please include the documentation changes in your PR. +6. Submit the PR. Once your PR is open, please do not force push to the branch containing your pull request to squash or amend commits. We use `squash merges `_ on PRs and rewriting commits makes changes harder to track between reviews. -6. Did your tests pass on Azure Pipelines? If they didn't, fix any errors. +7. Did your tests pass on Azure Pipelines? If they didn't, fix any errors. .. _ask for help: diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index c1d8cc403..c2d79dc33 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -44,17 +44,6 @@ supports `_ modern OSes based on Debian, Ubuntu, Fedora, SUSE, Gentoo and Darwin. - -Additional integrity verification of certbot-auto script can be done by verifying its digital signature. -This requires a local installation of gpg2, which comes packaged in many Linux distributions under name gnupg or gnupg2. - - -Installing with ``certbot-auto`` requires 512MB of RAM in order to build some -of the dependencies. Installing from pre-built OS packages avoids this -requirement. You can also temporarily set a swap file. See "Problems with -Python virtual environment" below for details. - - Alternate installation methods ================================ @@ -78,74 +67,6 @@ choosing "snapd" in the "System" dropdown menu. (You should select "snapd" regardless of your operating system, as our instructions are the same across all systems.) -.. _certbot-auto: - -Certbot-Auto ------------- - -The ``certbot-auto`` wrapper script installs Certbot, obtaining some dependencies -from your web server OS and putting others in a python virtual environment. You can -download and run it as follows:: - - wget https://dl.eff.org/certbot-auto - sudo mv certbot-auto /usr/local/bin/certbot-auto - sudo chown root /usr/local/bin/certbot-auto - sudo chmod 0755 /usr/local/bin/certbot-auto - /usr/local/bin/certbot-auto --help - -To remove certbot-auto, just delete it and the files it places under /opt/eff.org, along with any cronjob or systemd timer you may have created. - -To check the integrity of the ``certbot-auto`` script, -you can use these steps:: - - - user@webserver:~$ wget -N https://dl.eff.org/certbot-auto.asc - user@webserver:~$ gpg2 --keyserver pool.sks-keyservers.net --recv-key A2CFB51FA275A7286234E7B24D17C995CD9775F2 - user@webserver:~$ gpg2 --trusted-key 4D17C995CD9775F2 --verify certbot-auto.asc /usr/local/bin/certbot-auto - - - -The output of the last command should look something like:: - - - gpg: Signature made Wed 02 May 2018 05:29:12 AM IST - gpg: using RSA key A2CFB51FA275A7286234E7B24D17C995CD9775F2 - gpg: key 4D17C995CD9775F2 marked as ultimately trusted - gpg: checking the trustdb - gpg: marginals needed: 3 completes needed: 1 trust model: pgp - gpg: depth: 0 valid: 2 signed: 2 trust: 0-, 0q, 0n, 0m, 0f, 2u - gpg: depth: 1 valid: 2 signed: 0 trust: 2-, 0q, 0n, 0m, 0f, 0u - gpg: next trustdb check due at 2027-11-22 - gpg: Good signature from "Let's Encrypt Client Team " [ultimate] - - - -The ``certbot-auto`` command updates to the latest client release automatically. -Since ``certbot-auto`` is a wrapper to ``certbot``, it accepts exactly -the same command line flags and arguments. For more information, see -`Certbot command-line options `_. - -For full command line help, you can type:: - - /usr/local/bin/certbot-auto --help all - -Problems with Python virtual environment ----------------------------------------- - -On a low memory system such as VPS with less than 512MB of RAM, the required dependencies of Certbot will fail to build. -This can be identified if the pip outputs contains something like ``internal compiler error: Killed (program cc1)``. -You can workaround this restriction by creating a temporary swapfile:: - - user@webserver:~$ sudo fallocate -l 1G /tmp/swapfile - user@webserver:~$ sudo chmod 600 /tmp/swapfile - user@webserver:~$ sudo mkswap /tmp/swapfile - user@webserver:~$ sudo swapon /tmp/swapfile - -Disable and remove the swapfile once the virtual environment is constructed:: - - user@webserver:~$ sudo swapoff /tmp/swapfile - user@webserver:~$ sudo rm /tmp/swapfile - .. _docker-user: Running with Docker @@ -161,7 +82,7 @@ Docker if you are sure you know what you are doing and have a good reason to do so. You should definitely read the :ref:`where-certs` section, in order to -know how to manage the certs +know how to manage the certificates manually. `Our ciphersuites page `__ provides some information about recommended ciphersuites. If none of these make much sense to you, you should definitely use the installation method @@ -207,6 +128,18 @@ of the ``/etc/letsencrypt`` directory, see :ref:`where-certs`. Operating System Packages ------------------------- +.. warning:: While the Certbot team tries to keep the Certbot packages offered + by various operating systems working in the most basic sense, due to + distribution policies and/or the limited resources of distribution + maintainers, Certbot OS packages often have problems that other distribution + mechanisms do not. The packages are often old resulting in a lack of bug + fixes and features and a worse TLS configuration than is generated by newer + versions of Certbot. They also may not configure certificate renewal for you + or have all of Certbot's plugins available. For reasons like these, we + recommend most users follow the instructions at + https://certbot.eff.org/instructions and OS packages are only documented + here as an alternative. + **Arch Linux** .. code-block:: shell @@ -273,8 +206,8 @@ Optionally to install the Certbot Apache plugin, you can use: **Gentoo** -The official Certbot client is available in Gentoo Portage. From the -official Certbot plugins, three of them are also available in Portage. +The official Certbot client is available in Gentoo Portage. From the +official Certbot plugins, three of them are also available in Portage. They need to be installed separately if you require their functionality. .. code-block:: shell @@ -284,7 +217,7 @@ They need to be installed separately if you require their functionality. emerge -av app-crypt/certbot-nginx emerge -av app-crypt/certbot-dns-nsone -.. Note:: The ``app-crypt/certbot-dns-nsone`` package has a different +.. Note:: The ``app-crypt/certbot-dns-nsone`` package has a different maintainer than the other packages and can lag behind in version. **NetBSD** @@ -303,6 +236,35 @@ OS packaging is an ongoing effort. If you'd like to package Certbot for your distribution of choice please have a look at the :doc:`packaging`. +.. _certbot-auto: + +Certbot-Auto +------------ + +We used to have a shell script named ``certbot-auto`` to help people install +Certbot on UNIX operating systems, however, this script is no longer supported. +If you want to uninstall ``certbot-auto``, you can follow our instructions +:doc:`here `. + +Problems with Python virtual environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When using ``certbot-auto`` on a low memory system such as VPS with less than +512MB of RAM, the required dependencies of Certbot may fail to build. This can +be identified if the pip outputs contains something like ``internal compiler +error: Killed (program cc1)``. You can workaround this restriction by creating +a temporary swapfile:: + + user@webserver:~$ sudo fallocate -l 1G /tmp/swapfile + user@webserver:~$ sudo chmod 600 /tmp/swapfile + user@webserver:~$ sudo mkswap /tmp/swapfile + user@webserver:~$ sudo swapon /tmp/swapfile + +Disable and remove the swapfile once the virtual environment is constructed:: + + user@webserver:~$ sudo swapoff /tmp/swapfile + user@webserver:~$ sudo rm /tmp/swapfile + Installing from source ---------------------- diff --git a/certbot/docs/uninstall.rst b/certbot/docs/uninstall.rst new file mode 100644 index 000000000..65151242c --- /dev/null +++ b/certbot/docs/uninstall.rst @@ -0,0 +1,16 @@ +========================= +Uninstalling certbot-auto +========================= + +To uninstall ``certbot-auto``, you need to do three things: + +1. If you added a cron job or systemd timer to automatically run + ``certbot-auto`` to renew your certificates, you should delete it. If you + did this by following our instructions, you can delete the entry added to + ``/etc/crontab`` by running a command like ``sudo sed -i '/certbot-auto/d' + /etc/crontab``. +2. Delete the ``certbot-auto`` script. If you placed it in ``/usr/local/bin`` + like we recommended, you can delete it by running ``sudo rm + /usr/local/bin/certbot-auto``. +3. Delete the Certbot installation created by ``certbot-auto`` by running + ``sudo rm -rf /opt/eff.org``. diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 1912dafa4..ab8d64d79 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -179,10 +179,9 @@ If you'd like to obtain a wildcard certificate from Let's Encrypt or run Certbot's DNS plugins. These plugins are not included in a default Certbot installation and must be -installed separately. While the DNS plugins cannot currently be used with -``certbot-auto``, they are available in many OS package managers, as Docker -images, and as snaps. Visit https://certbot.eff.org to learn the best way to use -the DNS plugins on your system. +installed separately. They are available in many OS package managers, as Docker +images, and as snaps. Visit https://certbot.eff.org to learn the best way to +use the DNS plugins on your system. Once installed, you can find documentation on how to use each plugin at: @@ -314,7 +313,7 @@ the ``certificates`` subcommand: This returns information in the following format:: - Found the following certs: + Found the following certificates: Certificate Name: example.com Domains: example.com, www.example.com Expiry Date: 2017-02-19 19:53:00+00:00 (VALID: 30 days) @@ -913,7 +912,7 @@ Changing the ACME Server ======================== By default, Certbot uses Let's Encrypt's production server at -https://acme-v02.api.letsencrypt.org/. You can tell Certbot to use a +https://acme-v02.api.letsencrypt.org/directory. You can tell Certbot to use a different CA by providing ``--server`` on the command line or in a :ref:`configuration file ` with the URL of the server's ACME directory. For example, if you would like to use Let's Encrypt's diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f40689e57..f058cb658 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -13,7 +13,6 @@ except ImportError: # pragma: no cover from certbot import errors from certbot import util from certbot._internal import account -from certbot.compat import filesystem from certbot.compat import os import certbot.tests.util as test_util diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index 262fd02b5..263029cb0 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -1,7 +1,6 @@ """Tests for certbot.compat.filesystem""" import contextlib import errno -import stat import unittest try: diff --git a/certbot/tests/helpful_test.py b/certbot/tests/helpful_test.py index 292e55304..1a5c2bea6 100644 --- a/certbot/tests/helpful_test.py +++ b/certbot/tests/helpful_test.py @@ -1,6 +1,11 @@ """Tests for certbot.helpful_parser""" import unittest +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock + from certbot import errors from certbot._internal.cli import HelpfulArgumentParser from certbot._internal.cli import _DomainsAction @@ -189,5 +194,16 @@ class TestParseArgsErrors(unittest.TestCase): arg_parser.parse_args() +class TestAddDeprecatedArgument(unittest.TestCase): + """Tests for add_deprecated_argument method of HelpfulArgumentParser""" + + @mock.patch.object(HelpfulArgumentParser, "modify_kwargs_for_default_detection") + def test_no_default_detection_modifications(self, mock_modify): + arg_parser = HelpfulArgumentParser(["run"], {}, detect_defaults=True) + arg_parser.add_deprecated_argument("--foo", 0) + arg_parser.parse_args() + mock_modify.assert_not_called() + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 9be612c3b..18336776e 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1169,7 +1169,7 @@ class MainTest(test_util.ConfigTestCase): _, _, stdout = self._test_renewal_common(False, extra_args=None, should_renew=False, args=['renew'], expiry_date=expiry) self.assertTrue('No renewals were attempted.' in stdout.getvalue()) - self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue()) + self.assertTrue('The following certificates are not due for renewal yet:' in stdout.getvalue()) @mock.patch('certbot._internal.log.post_arg_parse_setup') def test_quiet_renew(self, _): @@ -1444,85 +1444,6 @@ class MainTest(test_util.ConfigTestCase): x = self._call_no_clientmock(["register", "--email", "user@example.org"]) self.assertTrue("There is an existing account" in x[0]) - def test_update_account_no_existing_accounts(self): - # with mock.patch('certbot._internal.main.client') as mocked_client: - with mock.patch('certbot._internal.main.account') as mocked_account: - mocked_storage = mock.MagicMock() - mocked_account.AccountFileStorage.return_value = mocked_storage - mocked_storage.find_all.return_value = [] - x = self._call_no_clientmock( - ["update_account", "--email", - "user@example.org"]) - self.assertTrue("Could not find an existing account" in x[0]) - - @mock.patch('certbot._internal.main._determine_account') - @mock.patch('certbot._internal.eff.prepare_subscription') - @mock.patch('certbot._internal.main.account') - def test_update_account_remove_email(self, mocked_account_module, mock_prepare, mock_det_acc): - # Mock account storage and the account object returned - mocked_storage = mock.MagicMock() - mocked_account = mock.MagicMock() - - mocked_account_module.AccountFileStorage.return_value = mocked_storage - mocked_storage.find_all.return_value = [mocked_account] - mock_det_acc.return_value = (mocked_account, "foo") - - # Mock registration body to verify calls are made - mock_regr_body = mock.MagicMock() - - # mocked_account.regr is overwritten in update, requiring an odd mock setup - mocked_account.regr.body = mock_regr_body - - x = self._call( - ["update_account", "--register-unsafely-without-email"]) - - - # When update succeeds, the return value of update_account() is None - self.assertTrue(x[0] is None) - # and we got supposedly did update the registration from - # the server - client_mock = x[3] - self.assertTrue(client_mock.Client().acme.update_registration.called) - - self.assertTrue(mock_regr_body.update.called) - self.assertTrue('contact' in mock_regr_body.update.call_args[1]) - self.assertEqual(mock_regr_body.update.call_args[1]['contact'], ()) - # and we saved the updated registration on disk - self.assertTrue(mocked_storage.update_regr.called) - # ensure we didn't try to subscribe (no email to subscribe with) - self.assertFalse(mock_prepare.called) - - @mock.patch("certbot._internal.main.display_util.notify") - @mock.patch('certbot._internal.main.display_ops.get_email') - @test_util.patch_get_utility() - def test_update_account_with_email(self, mock_utility, mock_email, mock_notify): - email = "user@example.com" - mock_email.return_value = email - with mock.patch('certbot._internal.eff.prepare_subscription') as mock_prepare: - with mock.patch('certbot._internal.main._determine_account') as mocked_det: - with mock.patch('certbot._internal.main.account') as mocked_account: - with mock.patch('certbot._internal.main.client') as mocked_client: - mocked_storage = mock.MagicMock() - mocked_account.AccountFileStorage.return_value = mocked_storage - mocked_storage.find_all.return_value = ["an account"] - mocked_det.return_value = (mock.MagicMock(), "foo") - cb_client = mock.MagicMock() - mocked_client.Client.return_value = cb_client - x = self._call_no_clientmock( - ["update_account"]) - # When registration change succeeds, the return value - # of register() is None - self.assertTrue(x[0] is None) - # and we got supposedly did update the registration from - # the server - self.assertTrue( - cb_client.acme.update_registration.called) - # and we saved the updated registration on disk - self.assertTrue(mocked_storage.update_regr.called) - self.assertTrue( - email in mock_notify.call_args[0][0]) - self.assertTrue(mock_prepare.called) - @mock.patch('certbot._internal.plugins.selection.choose_configurator_plugins') @mock.patch('certbot._internal.updater._run_updaters') def test_plugin_selection_error(self, mock_run, mock_choose): @@ -1795,5 +1716,111 @@ class InstallTest(test_util.ConfigTestCase): self.config, plugins) +class UpdateAccountTest(test_util.ConfigTestCase): + """Tests for certbot._internal.main.update_account""" + + def setUp(self): + patches = { + 'account': mock.patch('certbot._internal.main.account'), + 'atexit': mock.patch('certbot.util.atexit'), + 'client': mock.patch('certbot._internal.main.client'), + 'determine_account': mock.patch('certbot._internal.main._determine_account'), + 'notify': mock.patch('certbot._internal.main.display_util.notify'), + 'prepare_sub': mock.patch('certbot._internal.eff.prepare_subscription'), + 'util': test_util.patch_get_utility() + } + self.mocks = { k: patches[k].start() for k in patches } + for patch in patches.values(): + self.addCleanup(patch.stop) + + return super(UpdateAccountTest, self).setUp() + + def _call(self, args): + with mock.patch('certbot._internal.main.sys.stdout'), \ + mock.patch('certbot._internal.main.sys.stderr'): + args = ['--config-dir', self.config.config_dir, + '--work-dir', self.config.work_dir, + '--logs-dir', self.config.logs_dir, '--text'] + args + return main.main(args[:]) # NOTE: parser can alter its args! + + def _prepare_mock_account(self): + mock_storage = mock.MagicMock() + mock_account = mock.MagicMock() + mock_regr = mock.MagicMock() + mock_storage.find_all.return_value = [mock_account] + self.mocks['account'].AccountFileStorage.return_value = mock_storage + mock_account.regr.body = mock_regr.body + self.mocks['determine_account'].return_value = (mock_account, mock.MagicMock()) + return (mock_account, mock_storage, mock_regr) + + def _test_update_no_contact(self, args): + """Utility to assert that email removal is handled correctly""" + (_, mock_storage, mock_regr) = self._prepare_mock_account() + result = self._call(args) + # When update succeeds, the return value of update_account() is None + self.assertIsNone(result) + # We submitted a registration to the server + self.assertEqual(self.mocks['client'].Client().acme.update_registration.call_count, 1) + mock_regr.body.update.assert_called_with(contact=()) + # We got an update from the server and persisted it + self.assertEqual(mock_storage.update_regr.call_count, 1) + # We should have notified the user + self.mocks['notify'].assert_called_with( + 'Any contact information associated with this account has been removed.' + ) + # We should not have called subscription because there's no email + self.mocks['prepare_sub'].assert_not_called() + + def test_no_existing_accounts(self): + """Test that no existing account is handled correctly""" + mock_storage = mock.MagicMock() + mock_storage.find_all.return_value = [] + self.mocks['account'].AccountFileStorage.return_value = mock_storage + self.assertEqual(self._call(['update_account', '--email', 'user@example.org']), + 'Could not find an existing account to update.') + + def test_update_account_remove_email(self): + """Test that --register-unsafely-without-email is handled as no email""" + self._test_update_no_contact(['update_account', '--register-unsafely-without-email']) + + def test_update_account_empty_email(self): + """Test that providing an empty email is handled as no email""" + self._test_update_no_contact(['update_account', '-m', '']) + + @mock.patch('certbot._internal.main.display_ops.get_email') + def test_update_account_with_email(self, mock_email): + """Test that updating with a singular email is handled correctly""" + mock_email.return_value = 'user@example.com' + (_, mock_storage, _) = self._prepare_mock_account() + mock_client = mock.MagicMock() + self.mocks['client'].Client.return_value = mock_client + + result = self._call(['update_account']) + # None if registration succeeds + self.assertIsNone(result) + # We should have updated the server + self.assertEqual(mock_client.acme.update_registration.call_count, 1) + # We should have updated the account on disk + self.assertEqual(mock_storage.update_regr.call_count, 1) + # Subscription should have been prompted + self.assertEqual(self.mocks['prepare_sub'].call_count, 1) + # Should have printed the email + self.mocks['notify'].assert_called_with( + 'Your e-mail address was updated to user@example.com.') + + def test_update_account_with_multiple_emails(self): + """Test that multiple email addresses are handled correctly""" + (_, mock_storage, mock_regr) = self._prepare_mock_account() + self.assertIsNone( + self._call(['update_account', '-m', 'user@example.com,user@example.org']) + ) + mock_regr.body.update.assert_called_with( + contact=['mailto:user@example.com', 'mailto:user@example.org'] + ) + self.assertEqual(mock_storage.update_regr.call_count, 1) + self.mocks['notify'].assert_called_with( + 'Your e-mail address was updated to user@example.com,user@example.org.') + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index e292d24fb..4af8c6e7f 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -163,5 +163,70 @@ class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): self.assertEqual(self.config.server, constants.CLI_DEFAULTS['server']) +class DescribeResultsTest(unittest.TestCase): + """Tests for certbot._internal.renewal._renew_describe_results.""" + def setUp(self): + self.patchers = { + 'log_error': mock.patch('certbot._internal.renewal.logger.error'), + 'notify': mock.patch('certbot._internal.renewal.display_util.notify')} + self.mock_notify = self.patchers['notify'].start() + self.mock_error = self.patchers['log_error'].start() + + def tearDown(self): + for patch in self.patchers.values(): + patch.stop() + + @classmethod + def _call(cls, *args, **kwargs): + from certbot._internal.renewal import _renew_describe_results + _renew_describe_results(*args, **kwargs) + + def _assert_success_output(self, lines): + self.mock_notify.assert_has_calls([mock.call(l) for l in lines]) + + def test_no_renewal_attempts(self): + self._call(mock.MagicMock(dry_run=True), [], [], [], []) + self._assert_success_output(['No simulated renewals were attempted.']) + + def test_successful_renewal(self): + self._call(mock.MagicMock(dry_run=False), ['good.pem'], None, None, None) + self._assert_success_output([ + '\n- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + 'Congratulations, all renewals succeeded: ', + ' good.pem (success)', + '- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + ]) + + def test_failed_renewal(self): + self._call(mock.MagicMock(dry_run=False), [], ['bad.pem'], [], []) + self._assert_success_output([ + '\n- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + '- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + ]) + self.mock_error.assert_has_calls([ + mock.call('All %ss failed. The following certificates could not be renewed:', 'renewal'), + mock.call(' bad.pem (failure)'), + ]) + + def test_all_renewal(self): + self._call(mock.MagicMock(dry_run=True), + ['good.pem', 'good2.pem'], ['bad.pem', 'bad2.pem'], + ['foo.pem expires on 123'], ['errored.conf']) + self._assert_success_output([ + '\n- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + 'The following certificates are not due for renewal yet:', + ' foo.pem expires on 123 (skipped)', + 'The following simulated renewals succeeded:', + ' good.pem (success)\n good2.pem (success)\n', + '\nAdditionally, the following renewal configurations were invalid: ', + ' errored.conf (parsefail)', + '- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -', + ]) + self.mock_error.assert_has_calls([ + mock.call('The following %ss failed:', 'simulated renewal'), + mock.call(' bad.pem (failure)\n bad2.pem (failure)'), + ]) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 4b93d2c38..7b510fbb6 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -547,8 +547,7 @@ class OsInfoTest(unittest.TestCase): m_distro.linux_distribution.return_value = ("something", "else") self.assertEqual(cbutil.get_os_info(), ("something", "else")) - @mock.patch("certbot.util.subprocess.Popen") - def test_non_systemd_os_info(self, popen_mock): + def test_non_systemd_os_info(self): import certbot.util as cbutil with mock.patch('certbot.util._USE_DISTRO', False): with mock.patch('platform.system_alias', @@ -557,13 +556,14 @@ class OsInfoTest(unittest.TestCase): with mock.patch('platform.system_alias', return_value=('darwin', '', '')): - comm_mock = mock.Mock() - comm_attrs = {'communicate.return_value': - ('42.42.42', 'error')} - comm_mock.configure_mock(**comm_attrs) - popen_mock.return_value = comm_mock - self.assertEqual(cbutil.get_python_os_info()[0], 'darwin') - self.assertEqual(cbutil.get_python_os_info()[1], '42.42.42') + with mock.patch("subprocess.Popen") as popen_mock: + comm_mock = mock.Mock() + comm_attrs = {'communicate.return_value': + ('42.42.42', 'error')} + comm_mock.configure_mock(**comm_attrs) + popen_mock.return_value = comm_mock + self.assertEqual(cbutil.get_python_os_info()[0], 'darwin') + self.assertEqual(cbutil.get_python_os_info()[1], '42.42.42') with mock.patch('platform.system_alias', return_value=('freebsd', '9.3-RC3-p1', '')): diff --git a/letsencrypt-auto b/letsencrypt-auto index 2a0cda9b3..ee12d4706 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.9.0" +LE_AUTO_VERSION="1.10.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -799,11 +799,7 @@ BootstrapMageiaCommon() { # that function. If Bootstrap is set to a function that doesn't install any # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then - Bootstrap() { - BootstrapMessage "Debian-based OSes" - BootstrapDebCommon - } - BOOTSTRAP_VERSION="BootstrapDebCommon $BOOTSTRAP_DEB_COMMON_VERSION" + DEPRECATED_OS=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 @@ -1497,18 +1493,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.9.0 \ - --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \ - --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57 -acme==1.9.0 \ - --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \ - --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f -certbot-apache==1.9.0 \ - --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \ - --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e -certbot-nginx==1.9.0 \ - --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \ - --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65 +certbot==1.10.1 \ + --hash=sha256:011ac980fa21b9f29e02c9b8d8b86e8a4bf4670b51b6ad91656e401e9d2d2231 \ + --hash=sha256:0d9ee3fc09e0d03b2d1b1f1c4916e61ecfc6904b4216ddef4e6a5ca1424d9cb7 +acme==1.10.1 \ + --hash=sha256:752d598e54e98ad1e874de53fd50c61044f1b566d6deb790db5676ce9c573546 \ + --hash=sha256:fcbb559aedc96b404edf593e78517dcd7291984d5a37036c3fc77f3c5c122fd8 +certbot-apache==1.10.1 \ + --hash=sha256:f077b4b7f166627ef5e0921fe7cde57700670fc86e9ad9dbdfaf2c573cc0f2fa \ + --hash=sha256:97ed637b4c7b03820db6c69aa90145dc989933351d46a3d62baf6b71674f0a10 +certbot-nginx==1.10.1 \ + --hash=sha256:7c36459021f8a1ec3b6c062e4c4fc866bfaa1dbf26ccd29e043dd6848003be08 \ + --hash=sha256:c0bbeccf85f46b728fd95e6bb8c2649d32d3383d7f47ea4b9c312d12bf04d2f0 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/Dockerfile.redhat6 b/letsencrypt-auto-source/Dockerfile.redhat6 deleted file mode 100644 index 66f21bc14..000000000 --- a/letsencrypt-auto-source/Dockerfile.redhat6 +++ /dev/null @@ -1,54 +0,0 @@ -# For running tests, build a docker image with a passwordless sudo and a trust -# store we can manipulate. - -ARG REDHAT_DIST_FLAVOR -FROM ${REDHAT_DIST_FLAVOR}:6 - -ARG REDHAT_DIST_FLAVOR - -RUN curl -O https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm \ - && rpm -ivh epel-release-latest-6.noarch.rpm - -# Install pip and sudo: -RUN yum install -y python-pip sudo -# Update to a stable and tested version of pip. -# We do not use pipstrap here because it no longer supports Python 2.6. -RUN pip install pip==9.0.1 setuptools==29.0.1 wheel==0.29.0 -# Pin pytest version for increased stability -RUN pip install pytest==3.2.5 six==1.10.0 - -# Add an unprivileged user: -RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups wheel --uid 1000 lea - -# Let that user sudo: -RUN sed -i.bkp -e \ - 's/# %wheel\(NOPASSWD: ALL\)\?/%wheel/g' \ - /etc/sudoers - -RUN mkdir -p /home/lea/certbot - -# Install fake testing CA: -COPY ./tests/certs/ca/my-root-ca.crt.pem /usr/local/share/ca-certificates/ -RUN update-ca-trust - -# Copy current letsencrypt-auto: -COPY . /home/lea/certbot/letsencrypt-auto-source - -# Tweak uname binary for tests on fake 32bits -COPY tests/uname_wrapper.sh /bin -RUN mv /bin/uname /bin/uname_orig \ - && mv /bin/uname_wrapper.sh /bin/uname \ - && chmod +x /bin/uname - -# Fetch previous letsencrypt-auto that was installing python 3.4 -RUN curl https://raw.githubusercontent.com/certbot/certbot/v0.38.0/letsencrypt-auto-source/letsencrypt-auto \ - -o /home/lea/certbot/letsencrypt-auto-source/letsencrypt-auto_py_34 \ - && chmod +x /home/lea/certbot/letsencrypt-auto-source/letsencrypt-auto_py_34 - -RUN cp /home/lea/certbot/letsencrypt-auto-source/tests/${REDHAT_DIST_FLAVOR}6_tests.sh /home/lea/certbot/letsencrypt-auto-source/tests/redhat6_tests.sh \ - && chmod +x /home/lea/certbot/letsencrypt-auto-source/tests/redhat6_tests.sh - -USER lea -WORKDIR /home/lea - -CMD ["sudo", "certbot/letsencrypt-auto-source/tests/redhat6_tests.sh"] diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index f73e8b35e..c1897074c 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl98wk8ACgkQTRfJlc2X -dfIctgf/TO83xXJJ8haqxke0ehHCwcmipX7ijPhwvaUTSqciMa56KnGJLNp1lAVz -vv8sfHUf7NSvGlRg+5M0szWY25+JzveJDNzse3rOzFmxA1GNKUycE3/zE/IdBRwN -fmxJHaUBrBL2erBZPHe8gFGTvlzopBoGSmQpWGY3hIufPWKBJohCbTscKbaa9hyz -njmMvwRdeqzvLWVZ4jNDDsil9kKl2Emue3guzA/cvVxHe17DZyLDfqni7ysZIcTn -wPAQzpLBKHyiqVRoVk+BJ6Z6wamW4NAxKbjXy9GrHy4txlfW8tGd3jXha8yWqJeH -xEFK02Zp+T17+C5uqEW4o0cIofMjCw== -=9UGf +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl/JL3kACgkQTRfJlc2X +dfKJMwf/RXjfg5KScEjWiR+YMAcTVxGl4ITDMNBvmPoqCfrPwIJQewy1k6yQUITr +tMe0tkPneGgGccJreLAuO4+RdmNqm2MKBO3wMW9YZobJxcbMmrtVxyBD2OP4K/lL +oCZvjcN5pLvje6OlMwJ/fQ+zGY8mFUpfKIluxKrqkkO3p6Q+i/wPXF5Gjjb2J/bI +N+TczQJYUkDWAw7Tp4ho3J9xpqIn3zyOc2hI3wQDMC1o9sU5a80Vyc/mEqpE8SQ3 +qOWg9Gdx3DXTWOztcx2IxZtFEkIukPM8iD/Fkr//3XHeIc3+mqRAQdY+w7EopzbP +hLwjHVEJs1EMYq8ntWmMFjZ4+ImFgw== +=Peuv -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 83a1041ba..7f358f805 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.10.0.dev0" +LE_AUTO_VERSION="1.11.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -804,6 +804,7 @@ elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 elif [ -f /etc/redhat-release ]; then + DEPRECATED_OS=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -836,12 +837,7 @@ elif [ -f /etc/redhat-release ]; then INTERACTIVE_BOOTSTRAP=1 fi - Bootstrap() { - BootstrapMessage "Legacy RedHat-based OSes that will use Python3" - BootstrapRpmPython3Legacy - } USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION" # Try now to enable SCL rh-python36 for systems already bootstrapped # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto @@ -860,18 +856,7 @@ elif [ -f /etc/redhat-release ]; then fi if [ "$RPM_USE_PYTHON_3" = 1 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi fi @@ -889,10 +874,7 @@ elif uname | grep -iq FreeBSD ; then elif uname | grep -iq Darwin ; then DEPRECATED_OS=1 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then - Bootstrap() { - ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + DEPRECATED_OS=1 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then DEPRECATED_OS=1 else @@ -1493,18 +1475,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.9.0 \ - --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \ - --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57 -acme==1.9.0 \ - --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \ - --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f -certbot-apache==1.9.0 \ - --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \ - --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e -certbot-nginx==1.9.0 \ - --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \ - --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65 +certbot==1.10.1 \ + --hash=sha256:011ac980fa21b9f29e02c9b8d8b86e8a4bf4670b51b6ad91656e401e9d2d2231 \ + --hash=sha256:0d9ee3fc09e0d03b2d1b1f1c4916e61ecfc6904b4216ddef4e6a5ca1424d9cb7 +acme==1.10.1 \ + --hash=sha256:752d598e54e98ad1e874de53fd50c61044f1b566d6deb790db5676ce9c573546 \ + --hash=sha256:fcbb559aedc96b404edf593e78517dcd7291984d5a37036c3fc77f3c5c122fd8 +certbot-apache==1.10.1 \ + --hash=sha256:f077b4b7f166627ef5e0921fe7cde57700670fc86e9ad9dbdfaf2c573cc0f2fa \ + --hash=sha256:97ed637b4c7b03820db6c69aa90145dc989933351d46a3d62baf6b71674f0a10 +certbot-nginx==1.10.1 \ + --hash=sha256:7c36459021f8a1ec3b6c062e4c4fc866bfaa1dbf26ccd29e043dd6848003be08 \ + --hash=sha256:c0bbeccf85f46b728fd95e6bb8c2649d32d3383d7f47ea4b9c312d12bf04d2f0 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 66b2b2084..c701f4a4d 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 5eb82b705..bc27469fb 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -326,6 +326,7 @@ elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 elif [ -f /etc/redhat-release ]; then + DEPRECATED_OS=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -358,12 +359,7 @@ elif [ -f /etc/redhat-release ]; then INTERACTIVE_BOOTSTRAP=1 fi - Bootstrap() { - BootstrapMessage "Legacy RedHat-based OSes that will use Python3" - BootstrapRpmPython3Legacy - } USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION" # Try now to enable SCL rh-python36 for systems already bootstrapped # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto @@ -382,18 +378,7 @@ elif [ -f /etc/redhat-release ]; then fi if [ "$RPM_USE_PYTHON_3" = 1 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi fi @@ -411,10 +396,7 @@ elif uname | grep -iq FreeBSD ; then elif uname | grep -iq Darwin ; then DEPRECATED_OS=1 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then - Bootstrap() { - ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + DEPRECATED_OS=1 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then DEPRECATED_OS=1 else diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 2fd82eb26..4839d1e72 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==1.9.0 \ - --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \ - --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57 -acme==1.9.0 \ - --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \ - --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f -certbot-apache==1.9.0 \ - --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \ - --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e -certbot-nginx==1.9.0 \ - --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \ - --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65 +certbot==1.10.1 \ + --hash=sha256:011ac980fa21b9f29e02c9b8d8b86e8a4bf4670b51b6ad91656e401e9d2d2231 \ + --hash=sha256:0d9ee3fc09e0d03b2d1b1f1c4916e61ecfc6904b4216ddef4e6a5ca1424d9cb7 +acme==1.10.1 \ + --hash=sha256:752d598e54e98ad1e874de53fd50c61044f1b566d6deb790db5676ce9c573546 \ + --hash=sha256:fcbb559aedc96b404edf593e78517dcd7291984d5a37036c3fc77f3c5c122fd8 +certbot-apache==1.10.1 \ + --hash=sha256:f077b4b7f166627ef5e0921fe7cde57700670fc86e9ad9dbdfaf2c573cc0f2fa \ + --hash=sha256:97ed637b4c7b03820db6c69aa90145dc989933351d46a3d62baf6b71674f0a10 +certbot-nginx==1.10.1 \ + --hash=sha256:7c36459021f8a1ec3b6c062e4c4fc866bfaa1dbf26ccd29e043dd6848003be08 \ + --hash=sha256:c0bbeccf85f46b728fd95e6bb8c2649d32d3383d7f47ea4b9c312d12bf04d2f0 diff --git a/letsencrypt-auto-source/tests/__init__.py b/letsencrypt-auto-source/tests/__init__.py deleted file mode 100644 index 8a1613aa5..000000000 --- a/letsencrypt-auto-source/tests/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Tests for letsencrypt-auto - -Run these locally by saying... :: - - ./build.py && docker build -t lea . -f Dockerfile. && docker run --rm -t -i lea - -""" diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py deleted file mode 100644 index 805bb21af..000000000 --- a/letsencrypt-auto-source/tests/auto_test.py +++ /dev/null @@ -1,503 +0,0 @@ -"""Tests for letsencrypt-auto""" - -from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler -from contextlib import contextmanager -from functools import partial -from json import dumps -from os import chmod, environ, makedirs, stat -from os.path import abspath, dirname, exists, join -import re -from shutil import copy, rmtree -import socket -import ssl -from stat import S_IMODE, S_IRUSR, S_IWUSR, S_IXUSR, S_IWGRP, S_IWOTH -from subprocess import CalledProcessError, Popen, PIPE -import sys -from tempfile import mkdtemp -from threading import Thread -from unittest import TestCase - -from pytest import mark -from six.moves import xrange # pylint: disable=redefined-builtin - - -@mark.skip -def tests_dir(): - """Return a path to the "tests" directory.""" - return dirname(abspath(__file__)) - - -def copy_stable(src, dst): - """ - Copy letsencrypt-auto, and replace its current version to its equivalent stable one. - This is needed to test correctly the self-upgrade functionality. - """ - copy(src, dst) - with open(dst, 'r') as file: - filedata = file.read() - filedata = re.sub(r'LE_AUTO_VERSION="(.*)\.dev0"', r'LE_AUTO_VERSION="\1"', filedata) - with open(dst, 'w') as file: - file.write(filedata) - - -sys.path.insert(0, dirname(tests_dir())) -from build import build as build_le_auto - - -BOOTSTRAP_FILENAME = 'certbot-auto-bootstrap-version.txt' -"""Name of the file where certbot-auto saves its bootstrap version.""" - - -class RequestHandler(BaseHTTPRequestHandler): - """An HTTPS request handler which is quiet and serves a specific folder.""" - - def __init__(self, resources, *args, **kwargs): - """ - :arg resources: A dict of resource paths pointing to content bytes - - """ - self.resources = resources - BaseHTTPRequestHandler.__init__(self, *args, **kwargs) - - def log_message(self, format, *args): - """Don't log each request to the terminal.""" - - def do_GET(self): - """Serve a GET request.""" - content = self.send_head() - if content is not None: - self.wfile.write(content) - - def send_head(self): - """Common code for GET and HEAD commands - - This sends the response code and MIME headers and returns either a - bytestring of content or, if none is found, None. - - """ - path = self.path[1:] # Strip leading slash. - content = self.resources.get(path) - if content is None: - self.send_error(404, 'Path "%s" not found in self.resources' % path) - else: - self.send_response(200) - self.send_header('Content-type', 'text/plain') - self.send_header('Content-Length', str(len(content))) - self.end_headers() - return content - - -def server_and_port(resources): - """Return an unstarted HTTPS server and the port it will use.""" - # Find a port, and bind to it. I can't get the OS to close the socket - # promptly after we shut down the server, so we typically need to try - # a couple ports after the first test case. Setting - # TCPServer.allow_reuse_address = True seems to have nothing to do - # with this behavior. - worked = False - for port in xrange(4443, 4543): - try: - server = HTTPServer(('localhost', port), - partial(RequestHandler, resources)) - except socket.error: - pass - else: - worked = True - server.socket = ssl.wrap_socket( - server.socket, - certfile=join(tests_dir(), 'certs', 'localhost', 'server.pem'), - server_side=True) - break - if not worked: - raise RuntimeError("Couldn't find an unused socket for the testing HTTPS server.") - return server, port - - -@contextmanager -def serving(resources): - """Spin up a local HTTPS server, and yield its base URL. - - Use a self-signed cert generated as outlined by - https://coolaj86.com/articles/create-your-own-certificate-authority-for- - testing/. - - """ - server, port = server_and_port(resources) - thread = Thread(target=server.serve_forever) - try: - thread.start() - yield 'https://localhost:{port}/'.format(port=port) - finally: - server.shutdown() - thread.join() - - -LE_AUTO_PATH = join(dirname(tests_dir()), 'letsencrypt-auto') - - -@contextmanager -def temp_paths(): - """Creates and deletes paths for letsencrypt-auto and its venv.""" - dir = mkdtemp(prefix='le-test-') - try: - yield join(dir, 'letsencrypt-auto'), join(dir, 'venv') - finally: - rmtree(dir, ignore_errors=True) - - -def out_and_err(command, input=None, shell=False, env=None): - """Run a shell command, and return stderr and stdout as string. - - If the command returns nonzero, raise CalledProcessError. - - :arg command: A list of commandline args - :arg input: Data to pipe to stdin. Omit for none. - - Remaining args have the same meaning as for Popen. - - """ - process = Popen(command, - stdout=PIPE, - stdin=PIPE, - stderr=PIPE, - shell=shell, - env=env) - out, err = process.communicate(input=input) - status = process.poll() # same as in check_output(), though wait() sounds better - if status: - error = CalledProcessError(status, command) - error.output = out - print('stdout output was:') - print(out) - print('stderr output was:') - print(err) - raise error - return out, err - - -def signed(content, private_key_name='signing.key'): - """Return the signed SHA-256 hash of ``content``, using the given key file.""" - command = ['openssl', 'dgst', '-sha256', '-sign', - join(tests_dir(), private_key_name)] - out, err = out_and_err(command, input=content) - return out - - -def install_le_auto(contents, install_path): - """Install some given source code as the letsencrypt-auto script at the - root level of a virtualenv. - - :arg contents: The contents of the built letsencrypt-auto script - :arg install_path: The path where to install the script - - """ - with open(install_path, 'w') as le_auto: - le_auto.write(contents) - chmod(install_path, S_IRUSR | S_IXUSR) - - -def run_le_auto(le_auto_path, venv_dir, base_url=None, le_auto_args_str='--version', **kwargs): - """Run the prebuilt version of letsencrypt-auto, returning stdout and - stderr strings. - - If the command returns other than 0, raise CalledProcessError. - - """ - env = environ.copy() - d = dict(VENV_PATH=venv_dir, - NO_CERT_VERIFY='1', - **kwargs) - - if base_url is not None: - # URL to PyPI-style JSON that tell us the latest released version - # of LE: - d['LE_AUTO_JSON_URL'] = base_url + 'certbot/json' - # URL to dir containing letsencrypt-auto and letsencrypt-auto.sig: - d['LE_AUTO_DIR_TEMPLATE'] = base_url + '%s/' - # The public key corresponding to signing.key: - d['LE_AUTO_PUBLIC_KEY'] = """-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg -tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G -hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT -uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl -LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 -Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 -iQIDAQAB ------END PUBLIC KEY-----""" - - env.update(d) - - return out_and_err( - le_auto_path + ' ' + le_auto_args_str, - shell=True, - env=env) - - -def set_le_script_version(venv_dir, version): - """Tell the letsencrypt script to report a certain version. - - We actually replace the script with a dummy version that knows only how to - print its version. - - """ - letsencrypt_path = join(venv_dir, 'bin', 'letsencrypt') - with open(letsencrypt_path, 'w') as script: - script.write("#!/usr/bin/env python\n" - "from sys import stderr\n" - "stderr.write('letsencrypt %s\\n')" % version) - chmod(letsencrypt_path, S_IRUSR | S_IXUSR) - - -def sudo_chmod(path, mode): - """Runs `sudo chmod mode path`.""" - mode = oct(mode).replace('o', '') - out_and_err(['sudo', 'chmod', mode, path]) - - -class AutoTests(TestCase): - """Test the major branch points of letsencrypt-auto: - - * An le-auto upgrade is needed. - * An le-auto upgrade is not needed. - * There was an out-of-date LE script installed. - * There was a current LE script installed. - * There was no LE script installed (less important). - * Pip hash-verification passes. - * Pip has a hash mismatch. - * The OpenSSL sig matches. - * The OpenSSL sig mismatches. - - For tests which get to the end, we run merely ``letsencrypt --version``. - The functioning of the rest of the certbot script is covered by other - test suites. - - """ - NEW_LE_AUTO = build_le_auto( - version='99.9.9', - requirements='letsencrypt==99.9.9 --hash=sha256:1cc14d61ab424cdee446f51e50f1123f8482ec740587fe78626c933bba2873a0') - NEW_LE_AUTO_SIG = signed(NEW_LE_AUTO) - - def test_successes(self): - """Exercise most branches of letsencrypt-auto. - - They just happen to be the branches in which everything goes well. - - I violate my usual rule of having small, decoupled tests, because... - - 1. We shouldn't need to run a Cartesian product of the branches: the - phases run in separate shell processes, containing state leakage - pretty effectively. The only shared state is FS state, and it's - limited to a temp dir, assuming (if we dare) all functions properly. - 2. One combination of branches happens to set us up nicely for testing - the next, saving code. - - """ - with temp_paths() as (le_auto_path, venv_dir): - # This serves a PyPI page with a higher version, a GitHub-alike - # with a corresponding le-auto script, and a matching signature. - resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, - 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} - with serving(resources) as base_url: - run_letsencrypt_auto = partial( - run_le_auto, - le_auto_path, - venv_dir, - base_url, - PIP_FIND_LINKS=join(tests_dir(), - 'fake-letsencrypt', - 'dist')) - - # Test when a phase-1 upgrade is needed, there's no LE binary - # installed, and pip hashes verify: - install_le_auto(build_le_auto(version='50.0.0'), le_auto_path) - out, err = run_letsencrypt_auto() - self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+', - err.strip().splitlines()[-1])) - # Make a few assertions to test the validity of the next tests: - self.assertTrue('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) - - # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This - # conveniently sets us up to test the next 2 cases. - - # Test when neither phase-1 upgrade nor phase-2 upgrade is - # needed (probably a common case): - out, err = run_letsencrypt_auto() - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertFalse('Creating virtual environment...' in out) - - def test_phase2_upgrade(self): - """Test a phase-2 upgrade without a phase-1 upgrade.""" - resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, - 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} - with serving(resources) as base_url: - pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') - with temp_paths() as (le_auto_path, venv_dir): - install_le_auto(self.NEW_LE_AUTO, le_auto_path) - - # Create venv saving the correct bootstrap script version - out, err = run_le_auto(le_auto_path, venv_dir, base_url, - PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) - with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: - bootstrap_version = f.read() - - # Create a new venv with an old letsencrypt version - with temp_paths() as (le_auto_path, venv_dir): - venv_bin = join(venv_dir, 'bin') - makedirs(venv_bin) - set_le_script_version(venv_dir, '0.0.1') - with open(join(venv_dir, BOOTSTRAP_FILENAME), 'w') as f: - f.write(bootstrap_version) - - install_le_auto(self.NEW_LE_AUTO, le_auto_path) - out, err = run_le_auto(le_auto_path, venv_dir, base_url, - PIP_FIND_LINKS=pip_find_links) - - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) - - def test_openssl_failure(self): - """Make sure we stop if the openssl signature check fails.""" - with temp_paths() as (le_auto_path, venv_dir): - # Serve an unrelated hash signed with the good key (easier than - # making a bad key, and a mismatch is a mismatch): - resources = {'': 'certbot/', - 'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': build_le_auto(version='99.9.9'), - 'v99.9.9/letsencrypt-auto.sig': signed('something else')} - with serving(resources) as base_url: - copy_stable(LE_AUTO_PATH, le_auto_path) - try: - out, err = run_le_auto(le_auto_path, venv_dir, base_url) - except CalledProcessError as exc: - self.assertEqual(exc.returncode, 1) - self.assertTrue("Couldn't verify signature of downloaded " - "certbot-auto." in exc.output) - else: - print(out) - self.fail('Signature check on certbot-auto erroneously passed.') - - def test_pip_failure(self): - """Make sure pip stops us if there is a hash mismatch.""" - with temp_paths() as (le_auto_path, venv_dir): - resources = {'': 'certbot/', - 'certbot/json': dumps({'releases': {'99.9.9': None}})} - with serving(resources) as base_url: - # Build a le-auto script embedding a bad requirements file: - install_le_auto( - build_le_auto( - version='99.9.9', - requirements='configobj==5.0.6 --hash=sha256:badbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadb'), - le_auto_path) - try: - out, err = run_le_auto(le_auto_path, venv_dir, base_url) - except CalledProcessError as exc: - self.assertEqual(exc.returncode, 1) - self.assertTrue("THESE PACKAGES DO NOT MATCH THE HASHES " - "FROM THE REQUIREMENTS FILE" in exc.output) - self.assertFalse( - exists(venv_dir), - msg="The virtualenv was left around, even though " - "installation didn't succeed. We shouldn't do " - "this, as it foils our detection of whether we " - "need to recreate the virtualenv, which hinges " - "on the presence of $VENV_BIN/letsencrypt.") - else: - self.fail("Pip didn't detect a bad hash and stop the " - "installation.") - - def test_permissions_warnings(self): - """Make sure letsencrypt-auto properly warns about permissions problems.""" - # This test assumes that only the parent of the directory containing - # letsencrypt-auto (usually /tmp) may have permissions letsencrypt-auto - # considers insecure. - with temp_paths() as (le_auto_path, venv_dir): - le_auto_path = abspath(le_auto_path) - le_auto_dir = dirname(le_auto_path) - le_auto_dir_parent = dirname(le_auto_dir) - install_le_auto(self.NEW_LE_AUTO, le_auto_path) - - run_letsencrypt_auto = partial( - run_le_auto, le_auto_path, venv_dir, - le_auto_args_str='--install-only --no-self-upgrade', - PIP_FIND_LINKS=join(tests_dir(), 'fake-letsencrypt', 'dist')) - # Run letsencrypt-auto once with current permissions to avoid - # potential problems when the script tries to write to temporary - # directories. - run_letsencrypt_auto() - - le_auto_dir_mode = stat(le_auto_dir).st_mode - le_auto_dir_parent_mode = S_IMODE(stat(le_auto_dir_parent).st_mode) - try: - # Make letsencrypt-auto happy with the current permissions - chmod(le_auto_dir, S_IRUSR | S_IXUSR) - sudo_chmod(le_auto_dir_parent, 0o755) - - self._test_permissions_warnings_about_path(le_auto_path, run_letsencrypt_auto) - self._test_permissions_warnings_about_path(le_auto_dir, run_letsencrypt_auto) - finally: - chmod(le_auto_dir, le_auto_dir_mode) - sudo_chmod(le_auto_dir_parent, le_auto_dir_parent_mode) - - def _test_permissions_warnings_about_path(self, path, run_le_auto_func): - # Test that there are no problems with the current permissions - out, _ = run_le_auto_func() - self.assertFalse('insecure permissions' in out) - - stat_result = stat(path) - original_mode = stat_result.st_mode - - # Test world permissions - chmod(path, original_mode | S_IWOTH) - out, _ = run_le_auto_func() - self.assertTrue('insecure permissions' in out) - - # Test group permissions - if stat_result.st_gid >= 1000: - chmod(path, original_mode | S_IWGRP) - out, _ = run_le_auto_func() - self.assertTrue('insecure permissions' in out) - - # Test owner permissions - if stat_result.st_uid >= 1000: - chmod(path, original_mode | S_IWUSR) - out, _ = run_le_auto_func() - self.assertTrue('insecure permissions' in out) - - # Test that permissions were properly restored - chmod(path, original_mode) - out, _ = run_le_auto_func() - self.assertFalse('insecure permissions' in out) - - def test_disabled_permissions_warnings(self): - """Make sure that letsencrypt-auto permissions warnings can be disabled.""" - with temp_paths() as (le_auto_path, venv_dir): - le_auto_path = abspath(le_auto_path) - install_le_auto(self.NEW_LE_AUTO, le_auto_path) - - le_auto_args_str='--install-only --no-self-upgrade' - pip_links=join(tests_dir(), 'fake-letsencrypt', 'dist') - out, _ = run_le_auto(le_auto_path, venv_dir, - le_auto_args_str=le_auto_args_str, - PIP_FIND_LINKS=pip_links) - self.assertTrue('insecure permissions' in out) - - # Test that warnings are disabled when the script isn't run as - # root. - out, _ = run_le_auto(le_auto_path, venv_dir, - le_auto_args_str=le_auto_args_str, - LE_AUTO_SUDO='', - PIP_FIND_LINKS=pip_links) - self.assertFalse('insecure permissions' in out) - - # Test that --no-permissions-check disables warnings. - le_auto_args_str += ' --no-permissions-check' - out, _ = run_le_auto( - le_auto_path, venv_dir, - le_auto_args_str=le_auto_args_str, - PIP_FIND_LINKS=pip_links) - self.assertFalse('insecure permissions' in out) diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh deleted file mode 100644 index 8bdffec87..000000000 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash -set -e -# Start by making sure your system is up-to-date: -yum update -y >/dev/null -yum install -y centos-release-scl >/dev/null -yum install -y python27 >/dev/null 2>/dev/null - -LE_AUTO_PY_34="certbot/letsencrypt-auto-source/letsencrypt-auto_py_34" -LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto" - -# Last version of certbot-auto that was bootstraping Python 3.4 for CentOS 6 users -INITIAL_CERTBOT_VERSION_PY34="certbot 0.38.0" - -# we're going to modify env variables, so do this in a subshell -( -# ensure CentOS6 32bits is not supported anymore, and so certbot is not installed -export UNAME_FAKE_32BITS=true -if ! "$LE_AUTO" 2>&1 | grep -q "Certbot cannot be installed."; then - echo "ERROR: certbot-auto installed certbot on 32-bit CentOS." - exit 1 -fi -) - -echo "PASSED: On CentOS 6 32 bits, certbot-auto refused to install certbot." - -# we're going to modify env variables, so do this in a subshell -( - . /opt/rh/python27/enable - - # ensure python 3 isn't installed - if python3 --version 2> /dev/null; then - echo "ERROR: Python3 is already installed." - exit 1 - fi - - # ensure python2.7 is available - if ! python2.7 --version 2> /dev/null; then - echo "ERROR: Python2.7 is not available." - exit 1 - fi - - # bootstrap, but don't install python 3. - "$LE_AUTO" --no-self-upgrade -n --version > /dev/null 2> /dev/null - - # ensure python 3 isn't installed - if python3 --version 2> /dev/null; then - echo "ERROR: letsencrypt-auto installed Python3 even though Python2.7 is present." - exit 1 - fi - - echo "PASSED: Did not upgrade to Python3 when Python2.7 is present." -) - -# ensure python2.7 isn't available -if python2.7 --version 2> /dev/null; then - echo "ERROR: Python2.7 is still available." - exit 1 -fi - -# Skip self upgrade due to Python 3 not being available. -if ! "$LE_AUTO" 2>&1 | grep -q "WARNING: couldn't find Python"; then - echo "ERROR: Python upgrade failure warning not printed!" - exit 1 -fi - -# bootstrap from the old letsencrypt-auto, this time installing python3.4 -"$LE_AUTO_PY_34" --no-self-upgrade -n --version >/dev/null 2>/dev/null - -# ensure python 3.4 is installed -if ! python3.4 --version >/dev/null 2>/dev/null; then - echo "ERROR: letsencrypt-auto failed to install Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present." - exit 1 -fi - -echo "PASSED: Successfully upgraded to Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present." - -# As "certbot-auto" (so without implicit --non-interactive flag set), check that the script -# refuses to install SCL Python 3.6 when run in a non interactive shell (simulated here -# using | tee /dev/null) if --non-interactive flag is not provided. -cp "$LE_AUTO" /tmp/certbot-auto -# NB: Readline has an issue on all Python versions for CentOS 6, making `certbot --version` -# output an unprintable ASCII character on a new line at the end. -# So we take the second last line of the output. -version=$(/tmp/certbot-auto --version 2>/dev/null | tee /dev/null | tail -2 | head -1) - -if [ "$version" != "$INITIAL_CERTBOT_VERSION_PY34" ]; then - echo "ERROR: certbot-auto upgraded certbot in a non-interactive shell with --non-interactive flag not set." - exit 1 -fi - -echo "PASSED: certbot-auto did not upgrade certbot in a non-interactive shell with --non-interactive flag not set." - -if [ -f /opt/rh/rh-python36/enable ]; then - echo "ERROR: certbot-auto installed Python3.6 in a non-interactive shell with --non-interactive flag not set." - exit 1 -fi - -echo "PASSED: certbot-auto did not install Python3.6 in a non-interactive shell with --non-interactive flag not set." - -# now bootstrap from current letsencrypt-auto, that will install python3.6 from SCL -"$LE_AUTO" --no-self-upgrade -n --version >/dev/null 2>/dev/null - -# Following test is executed in a subshell, to not leak any environment variable -( - # enable SCL rh-python36 - . /opt/rh/rh-python36/enable - - # ensure python 3.6 is installed - if ! python3.6 --version >/dev/null 2>/dev/null; then - echo "ERROR: letsencrypt-auto failed to install Python3.6 using current letsencrypt-auto when only Python2.6/Python3.4 are present." - exit 1 - fi - - echo "PASSED: Successfully upgraded to Python3.6 using current letsencrypt-auto when only Python2.6/Python3.4 are present." -) - -# Following test is executed in a subshell, to not leak any environment variable -( - export VENV_PATH=$(mktemp -d) - "$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1 - if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1-2)" != "3.6" ]; then - echo "ERROR: Python 3.6 wasn't used with --no-bootstrap!" - exit 1 - fi -) - -# Following test is executed in a subshell, to not leak any environment variable -( - # enable SCL rh-python36 - . /opt/rh/rh-python36/enable - - # ensure everything works fine with certbot-auto bootstrap when python 3.6 is already enabled - export VENV_PATH=$(mktemp -d) - if ! "$LE_AUTO" --no-self-upgrade -n --version >/dev/null 2>/dev/null; then - echo "ERROR: Certbot-auto broke when Python 3.6 SCL is already enabled." - exit 1 - fi -) - -# we're going to modify env variables, so do this in a subshell -( - # ensure CentOS6 32bits is not supported anymore, and so certbot - # is not upgraded nor reinstalled. - export UNAME_FAKE_32BITS=true - OUTPUT=$("$LE_AUTO" --version 2>&1) - if ! echo "$OUTPUT" | grep -q "Certbot will no longer receive updates."; then - echo "ERROR: certbot-auto failed to run or upgraded pre-existing Certbot instance on 32-bit CentOS 6." - exit 1 - fi - if ! "$LE_AUTO" --install-only 2>&1 | grep -q "Certbot cannot be installed."; then - echo "ERROR: certbot-auto reinstalled Certbot on 32-bit CentOS 6." - exit 1 - fi -) - -# we're going to modify env variables, so do this in a subshell -( - # Prepare a certbot installation in the old venv path - rm -rf /opt/eff.org - VENV_PATH=~/.local/share/letsencrypt "$LE_AUTO" --install-only > /dev/null 2> /dev/null - # fake 32 bits mode - export UNAME_FAKE_32BITS=true - OUTPUT=$("$LE_AUTO" --version 2>&1) - if ! echo "$OUTPUT" | grep -q "Certbot will no longer receive updates."; then - echo "ERROR: certbot-auto failed to run or upgraded pre-existing Certbot instance in the old venv path on 32-bit CentOS 6." - exit 1 - fi -) - -echo "PASSED: certbot-auto refused to install/upgrade certbot on 32-bit CentOS 6." - -# test using python3 -pytest -v -s certbot/letsencrypt-auto-source/tests diff --git a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem deleted file mode 100644 index 4e4d29bd2..000000000 --- a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem +++ /dev/null @@ -1,23 +0,0 @@ ------BEGIN CERTIFICATE----- -MIID5jCCAs6gAwIBAgIJAI1Qkfyw88REMA0GCSqGSIb3DQEBBQUAMFUxCzAJBgNV -BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMRswGQYDVQQKExJNeSBCb2d1cyBS -b290IENlcnQxFDASBgNVBAMTC2V4YW1wbGUuY29tMB4XDTE1MTIwNDIwNTIxNVoX -DTQwMTIwMzIwNTIxNVowVTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3Rh -dGUxGzAZBgNVBAoTEk15IEJvZ3VzIFJvb3QgQ2VydDEUMBIGA1UEAxMLZXhhbXBs -ZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQVQpQ2EH4gTJB -NJP6+ocT3xJwT8mSXYUnvzjj6iv+JxZiXRGzAPziNzrrSRKY0yDHF+UiJwuOerLa -n8laZkLb1Ogqzs2u64rKeb0xWv90Qp+eXG0J/1xb4dw+GExqe5QFo1JUJzO/eK7m -1S04SeFkN1qV9mD5yJUy7DGiTUzDHgCxM2tXMLusXYqkxsQQ9+2EJ7BEOK4YJGEx -Sign5FuSxb64PiNow6OA97CaLl7tV4INP4w195ueDRIaS4poeOep4s8U7IAdMjIZ -EryJgKNCij50xK92vPBBJSj0NOitltBlwoEqkOZpQCOZamFd6nvt78LQ6W8Am+l6 -y6oCON5JAgMBAAGjgbgwgbUwHQYDVR0OBBYEFAlrdStDhaayLLj89Whe3Gc+HE8y -MIGFBgNVHSMEfjB8gBQJa3UrQ4Wmsiy4/PVoXtxnPhxPMqFZpFcwVTELMAkGA1UE -BhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxGzAZBgNVBAoTEk15IEJvZ3VzIFJv -b3QgQ2VydDEUMBIGA1UEAxMLZXhhbXBsZS5jb22CCQCNUJH8sPPERDAMBgNVHRME -BTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQC7KAQfDTiNM3QO8Ic3x21CAPJUavkH -zshifN+Ei0+nmseHDTCTgsGfGDOToLUpUEZ4PuiHnz08UwRfd9wotc3SgY9ZaXMe -vRs8KUAF9EoyTvESzPyv2b6cS9NNMpj5y7KyXSyP17VoGbNavtiGQ4dwgEH6VgNl -0RtBvcSBv/tqxIIx1tWzL74tVEm0Kbd9BAZsYpQNKL8e6WXP35/j0PvCCvtofGrA -E8LTqMz4kCwnX+QaJIMJhBophRCsjXdAkvFbFxX0DGPztQtzIwBPcdMjsft7AFeE -0XchhDDXxw8YsbpvPfCvrD8XiiVuBycbnB1zt0LLVwB/QsCzUW9ImpLC ------END CERTIFICATE----- diff --git a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem deleted file mode 100644 index 9caa7ddaa..000000000 --- a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA0FUKUNhB+IEyQTST+vqHE98ScE/Jkl2FJ7844+or/icWYl0R -swD84jc660kSmNMgxxflIicLjnqy2p/JWmZC29ToKs7NruuKynm9MVr/dEKfnlxt -Cf9cW+HcPhhManuUBaNSVCczv3iu5tUtOEnhZDdalfZg+ciVMuwxok1Mwx4AsTNr -VzC7rF2KpMbEEPfthCewRDiuGCRhMUooJ+RbksW+uD4jaMOjgPewmi5e7VeCDT+M -Nfebng0SGkuKaHjnqeLPFOyAHTIyGRK8iYCjQoo+dMSvdrzwQSUo9DTorZbQZcKB -KpDmaUAjmWphXep77e/C0OlvAJvpesuqAjjeSQIDAQABAoIBAH+qbVzneV3wxjwh -HUHi/p3VyHXc3xh7iNq3mwRH/1eK2nPCttLsGwwBbnC64dOXJfH7maWZKcLRPAMv -gfOM0RHn4bJB8tdrbizv91lke0DihvBDkWpb+1wvB4lh2Io0Wpwt3ojFUTfXm87G -+iQRWjbQmQlm5zyKh6uiBDSCjDTQdb9omZEBMAwlGPTZwt8TRUEtWd8QgW8FCHoB -iLER2WBwXdvn3PBtocI3VE6IYDSeZ81Xv+d7925RtVintT8Suk4toYwX+jfSz+wZ -sgHd5V6PSv9a7GUlWoUihD99D9wqDZE8IvMDZ5ofSAUd1KfICDtmsEyugY7u2yYZ -tYt49AECgYEA73f7ITMHg8JsUipqb6eG10gCRtRhkqrrO1g/TNeTBh3CTrQGb56e -y6kmUivn5gK46t3T2N4Ht4IR8fpLcJcbPYPQNulSjmWm5y6WduafXW/VCW1NA9Lc -FyGPkMxFCIVJTLFxfLFepBVvtUzLLDKGGtQxru/GNbBzjdtmVfDPIoECgYEA3rbM -cTfvj+jWrV1YsRbphyjy+k3OJEIVx6KA4s5d7Tp12UfYQp/B3HPhXXm5wqeo1Nos -UAEWZIMi1VoE8iu6jjeJ6uERtbKKQVed25Us/ff0jUPbxlXgiBOtRcllq9d9Srjm -ybHUgfjLsZ2/xpIcOl+oI5pDM9JvD8Sq4ZCFR8kCgYBK/H0tFjeiML2OtS2DLShy -PWBJIbQ0I0Vp3eZkf5TQc30m/ASP61G6YItZa9pAElYpZbEy1cQA2MAZz9DTvt2O -07ndmA57/KTY+6OuM+Vvctd5DjrxmZPFwoKcSvrLAkHDvETXUQtbwkKquRNeEawg -tpWgPAELSufEYhGXk8KpAQKBgBDCqPgMQZcO6rj5QWdyVfi5+C8mE9Fet8ziSdjH -twHXWG8VnQzGgQxaHCewtW4Ut/vsv1D2A/1kcQalU6H18IArZdGrRm3qFcV9FoAj -5dLnChxncu6mH9Odx3htA52/BcrNx3B+VYPCeXHQcVI8RKuP71NelJgdygXhwwpe -mekhAoGBAOUovnqylciYa9HRqo+xZk59eyX+ehhnlV8SeJ2K0PwaQkzQ0KYtCmE7 -kdSdhcv8h/IQKGaFfc/LyFMM/a26PfAeY5bj41UjkT0K5hQrYuL/52xaT401YLcb -Xo+bZz9K0hrdP7TdZFuTY/WxojXgjsVAuAN1NwnJumqxhzPh+hfl ------END RSA PRIVATE KEY----- diff --git a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl deleted file mode 100644 index ad6d262b4..000000000 --- a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl +++ /dev/null @@ -1 +0,0 @@ -D613482D0EF95DD0 diff --git a/letsencrypt-auto-source/tests/certs/localhost/cert.pem b/letsencrypt-auto-source/tests/certs/localhost/cert.pem deleted file mode 100644 index ac83535ce..000000000 --- a/letsencrypt-auto-source/tests/certs/localhost/cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDKjCCAhICCQDWE0gtDvld0DANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJB -VTETMBEGA1UECBMKU29tZS1TdGF0ZTEbMBkGA1UEChMSTXkgQm9ndXMgUm9vdCBD -ZXJ0MRQwEgYDVQQDEwtleGFtcGxlLmNvbTAeFw0xNTEyMDQyMDU0MzFaFw00MDEy -MDMyMDU0MzFaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEw -HwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCWxvY2Fs -aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2WIIi86Mis4UQH -a5PrFbX2PBtQHbI3t3ekN1CewRsgQ/2X3lCeWhKmr3CJYXVgA7q/23PORQAiuV6y -DG2dQIrjeahWCXaCptTi49ljfVRTW2IxrHke/iA8TkDuZbWGzVLb8TB83ipBOD41 -SjuomoN4A/ktnIfbNqRqgjjHs2wwJHDfxPiCQlwyOayjHmdlh8cqfVE8rWEm5/3T -Iu0X1J53SammR1SbUmsLJNofxFYMK1ogHb0CaFEG9QuuUDPJl5K74Rr6InMQZKPn -ne4W3cGoALxPHAca7yicpSMSmdsmd6pqylc2Fdua7o/wf0SwShxS4A1DqA/HWLEM -V6MSEF8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAz5sMAFG6W/ZEULZITkBTCU6P -NttpGiKufnqyBW5HyNylaczfnHnClvQjr8f/84xvKVcfC3xP0lz+92aIQqo+5L/n -v7gLhBFR4Vr2XwMt2qz2FpkaxmVwnhVAHaaC05WIKQ6W2gDwWT0u1K8YdTh+7mvN -AT9FW4vDgtNZWq4W/PePh9QCiOOQhGOuBYj/7zqLtz4XPifhi66ILIRDHiu0kond -3YMFcECIAf4MPT9vT0iNcWX+c8CfAixPt8nMD6bzOo3oTcfuZh/2enfgLbMqOlOi -uk72FM5VVPXTWAckJvL/vVjqsvDuJQKqbr0oUc3bdWbS36xtWZUycp4IQLguAQ== ------END CERTIFICATE----- diff --git a/letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem b/letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem deleted file mode 100644 index 8a6189f88..000000000 --- a/letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICnjCCAYYCAQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUx -ITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAxMJbG9j -YWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZYgiLzoyKzh -RAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/ZfeUJ5aEqavcIlhdWADur/bc85FACK5 -XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGseR7+IDxOQO5ltYbNUtvxMHzeKkE4 -PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E+IJCXDI5rKMeZ2WHxyp9UTytYSbn -/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAdvQJoUQb1C65QM8mXkrvhGvoicxBk -o+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrKVzYV25ruj/B/RLBKHFLgDUOoD8dY -sQxXoxIQXwIDAQABoAAwDQYJKoZIhvcNAQEFBQADggEBAFbg3WrAokoPx7iAYG6z -PqeDd4/XanXjeL4Ryxv6LoGhu69mmBAd3N5ILPyQJjnkWpIjEmJDzEcPMzhQjRh5 -GlWTyvKWO4zClYU840KZk7crVkpzNZ+HP0YeM/Agz6sab00ffRcq5m1wEF9MCvDE -8FUXk1HBHRAb/6t9QV/7axsPOkGT8SjQ1v2SCaiB0HQL3sYChYLi5zu4dfmQNPGq -ar9Xm5a0YqOQIFfmy8RSwxk0Q/ipNFTGN1uvlIRkgbT9zPnodxjWZsSI9BF+q5Af -uiE/oAk7MxfJ0LyLfhOWB+T98bKIOVtFT3wMLS1IIgMogwqCEXFf30Q9p2iTEzqT -6UE= ------END CERTIFICATE REQUEST----- diff --git a/letsencrypt-auto-source/tests/certs/localhost/privkey.pem b/letsencrypt-auto-source/tests/certs/localhost/privkey.pem deleted file mode 100644 index 18feba403..000000000 --- a/letsencrypt-auto-source/tests/certs/localhost/privkey.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEArZYgiLzoyKzhRAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/Zfe -UJ5aEqavcIlhdWADur/bc85FACK5XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGs -eR7+IDxOQO5ltYbNUtvxMHzeKkE4PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E -+IJCXDI5rKMeZ2WHxyp9UTytYSbn/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAd -vQJoUQb1C65QM8mXkrvhGvoicxBko+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrK -VzYV25ruj/B/RLBKHFLgDUOoD8dYsQxXoxIQXwIDAQABAoIBAG8bVJ+xKt6nqVg9 -16HKKw9ZGIfy888K0qgFuFImCzwtntdGycmYUdb2Uf0aMgNK/ZgfDXxGXuwDTdtK -46GVsaY0i74vs8bjQZ2pzGVsxN+gqzFi0h6Es+w2LXBqJzfVnL6YgPykMB+jtzg6 -K9Wbyaq0uvZXN4XNzl/WvJtTV4i7Cff1MOd5EhKFdqxrZvB/SRBCr/SMMafRtB9P -EvMneNKzhmlrutHAxuyxEKZR32Kkx7ydAdTjGgn+rE+NL5BweXfeWhLU4Bv14bn9 -Mkneu3w5o1ryJfE2YnVajUP//jeopUT0nTQ3MpEusBQCLBlvFXjjM9uCaFX+5+MP -0H4xVcECgYEA1Q+wR3GHbk37vIGSlbENyUsri5WlMt8IVAHsDsTOpxAjYB0yyo+x -h9RS+RJZQECJlA6H72peUl3GM7RgdWIcKOT3nZ12XqYKG57rr/N5zlUuxbdS8KBk -JhyZeJdYjq/Jrno1ZP+OSmc7VvBLcM7irY7LHlvK0o8W1W0TNJ8jrZkCgYEA0JHX -lJd+fiezcUS7g4moHtzJp0JKquQiXLX+c2urmpyhb3ZrTuQ8OUjSy6DlwHlgDx8K -Hg2sdx/ZCuDaGjR4IY/Qs5RFt9WUqlK9gi9V3nYVrzBOQkdFOf/Ad3j4pQ8/aeCX -nP6snHXz1WqPpbCXG6l6GzFGbQU473GfuKsDuLcCgYAWQaNKc0OQdDj9whNL68ji -5CVSWXl+TOoTzHeaO1jS/s6TNbmei1AiPj3EovQL0DIO802j5tqfhAg2UntZB7yl -UPXE0zQQQwv/QqSgJrDsqt1N7g6N8FNF3+rwO+8WSKqqvT1ipYd5ojsCo+tdh18K -fkYdj70qLaRW+yPsdUtG0QKBgEYc8NqbvsML94+ZKmwCh4iwcf2PFGi0PjTqXTpR -tKNKCh7dMR+ZLAGZ0HrxgKqeYsNSjOUjdZmqFB1LDyaGAuhNXzwvGOy+mLZVEC3G -Wdhp28pDs9sl+EiSCBJhkTxzjr656F23YzFJmYlhxB5P6cw7wbeIbgNSIRylFqtO -mfarAoGBAICsAEWypOctxtmtOcjxgJ7jMbOA7rrsGlXpiy1/WlwIwRGF5LMvIIFX -qFAfiPcZn05ZgdAGzaFYowdjmQB10FW0jZbDf+nIHfOF5YmfmfWjsaweEGALJmqB -okGu/lGNGf3XoYzy0/hC3WAqk3znSZtQLUq8jEWF7dLNUizUeUow ------END RSA PRIVATE KEY----- diff --git a/letsencrypt-auto-source/tests/certs/localhost/server.pem b/letsencrypt-auto-source/tests/certs/localhost/server.pem deleted file mode 100644 index c5765dd89..000000000 --- a/letsencrypt-auto-source/tests/certs/localhost/server.pem +++ /dev/null @@ -1,46 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEArZYgiLzoyKzhRAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/Zfe -UJ5aEqavcIlhdWADur/bc85FACK5XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGs -eR7+IDxOQO5ltYbNUtvxMHzeKkE4PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E -+IJCXDI5rKMeZ2WHxyp9UTytYSbn/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAd -vQJoUQb1C65QM8mXkrvhGvoicxBko+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrK -VzYV25ruj/B/RLBKHFLgDUOoD8dYsQxXoxIQXwIDAQABAoIBAG8bVJ+xKt6nqVg9 -16HKKw9ZGIfy888K0qgFuFImCzwtntdGycmYUdb2Uf0aMgNK/ZgfDXxGXuwDTdtK -46GVsaY0i74vs8bjQZ2pzGVsxN+gqzFi0h6Es+w2LXBqJzfVnL6YgPykMB+jtzg6 -K9Wbyaq0uvZXN4XNzl/WvJtTV4i7Cff1MOd5EhKFdqxrZvB/SRBCr/SMMafRtB9P -EvMneNKzhmlrutHAxuyxEKZR32Kkx7ydAdTjGgn+rE+NL5BweXfeWhLU4Bv14bn9 -Mkneu3w5o1ryJfE2YnVajUP//jeopUT0nTQ3MpEusBQCLBlvFXjjM9uCaFX+5+MP -0H4xVcECgYEA1Q+wR3GHbk37vIGSlbENyUsri5WlMt8IVAHsDsTOpxAjYB0yyo+x -h9RS+RJZQECJlA6H72peUl3GM7RgdWIcKOT3nZ12XqYKG57rr/N5zlUuxbdS8KBk -JhyZeJdYjq/Jrno1ZP+OSmc7VvBLcM7irY7LHlvK0o8W1W0TNJ8jrZkCgYEA0JHX -lJd+fiezcUS7g4moHtzJp0JKquQiXLX+c2urmpyhb3ZrTuQ8OUjSy6DlwHlgDx8K -Hg2sdx/ZCuDaGjR4IY/Qs5RFt9WUqlK9gi9V3nYVrzBOQkdFOf/Ad3j4pQ8/aeCX -nP6snHXz1WqPpbCXG6l6GzFGbQU473GfuKsDuLcCgYAWQaNKc0OQdDj9whNL68ji -5CVSWXl+TOoTzHeaO1jS/s6TNbmei1AiPj3EovQL0DIO802j5tqfhAg2UntZB7yl -UPXE0zQQQwv/QqSgJrDsqt1N7g6N8FNF3+rwO+8WSKqqvT1ipYd5ojsCo+tdh18K -fkYdj70qLaRW+yPsdUtG0QKBgEYc8NqbvsML94+ZKmwCh4iwcf2PFGi0PjTqXTpR -tKNKCh7dMR+ZLAGZ0HrxgKqeYsNSjOUjdZmqFB1LDyaGAuhNXzwvGOy+mLZVEC3G -Wdhp28pDs9sl+EiSCBJhkTxzjr656F23YzFJmYlhxB5P6cw7wbeIbgNSIRylFqtO -mfarAoGBAICsAEWypOctxtmtOcjxgJ7jMbOA7rrsGlXpiy1/WlwIwRGF5LMvIIFX -qFAfiPcZn05ZgdAGzaFYowdjmQB10FW0jZbDf+nIHfOF5YmfmfWjsaweEGALJmqB -okGu/lGNGf3XoYzy0/hC3WAqk3znSZtQLUq8jEWF7dLNUizUeUow ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDKjCCAhICCQDWE0gtDvld0DANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJB -VTETMBEGA1UECBMKU29tZS1TdGF0ZTEbMBkGA1UEChMSTXkgQm9ndXMgUm9vdCBD -ZXJ0MRQwEgYDVQQDEwtleGFtcGxlLmNvbTAeFw0xNTEyMDQyMDU0MzFaFw00MDEy -MDMyMDU0MzFaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEw -HwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCWxvY2Fs -aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2WIIi86Mis4UQH -a5PrFbX2PBtQHbI3t3ekN1CewRsgQ/2X3lCeWhKmr3CJYXVgA7q/23PORQAiuV6y -DG2dQIrjeahWCXaCptTi49ljfVRTW2IxrHke/iA8TkDuZbWGzVLb8TB83ipBOD41 -SjuomoN4A/ktnIfbNqRqgjjHs2wwJHDfxPiCQlwyOayjHmdlh8cqfVE8rWEm5/3T -Iu0X1J53SammR1SbUmsLJNofxFYMK1ogHb0CaFEG9QuuUDPJl5K74Rr6InMQZKPn -ne4W3cGoALxPHAca7yicpSMSmdsmd6pqylc2Fdua7o/wf0SwShxS4A1DqA/HWLEM -V6MSEF8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAz5sMAFG6W/ZEULZITkBTCU6P -NttpGiKufnqyBW5HyNylaczfnHnClvQjr8f/84xvKVcfC3xP0lz+92aIQqo+5L/n -v7gLhBFR4Vr2XwMt2qz2FpkaxmVwnhVAHaaC05WIKQ6W2gDwWT0u1K8YdTh+7mvN -AT9FW4vDgtNZWq4W/PePh9QCiOOQhGOuBYj/7zqLtz4XPifhi66ILIRDHiu0kond -3YMFcECIAf4MPT9vT0iNcWX+c8CfAixPt8nMD6bzOo3oTcfuZh/2enfgLbMqOlOi -uk72FM5VVPXTWAckJvL/vVjqsvDuJQKqbr0oUc3bdWbS36xtWZUycp4IQLguAQ== ------END CERTIFICATE----- diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz b/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz deleted file mode 100644 index 5f9a48a34..000000000 Binary files a/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz and /dev/null differ diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py b/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py deleted file mode 100755 index 9d811fab5..000000000 --- a/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py +++ /dev/null @@ -1,8 +0,0 @@ -from sys import argv, stderr - - -def main(): - """Act like letsencrypt --version insofar as printing the version number to - stderr.""" - if '--version' in argv: - stderr.write('letsencrypt 99.9.9\n') diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py b/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py deleted file mode 100644 index e5f7fde35..000000000 --- a/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import setup - - -setup( - name='letsencrypt', - version='99.9.9', - description='A mock version of letsencrypt that just prints its version', - py_modules=['letsencrypt'], - entry_points={ - 'console_scripts': ['letsencrypt = letsencrypt:main'] - } -) diff --git a/letsencrypt-auto-source/tests/oraclelinux6_tests.sh b/letsencrypt-auto-source/tests/oraclelinux6_tests.sh deleted file mode 100644 index f3fd952f3..000000000 --- a/letsencrypt-auto-source/tests/oraclelinux6_tests.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -set -eo pipefail -# Start by making sure your system is up-to-date: -yum update -y >/dev/null - -LE_AUTO_PY_34="certbot/letsencrypt-auto-source/letsencrypt-auto_py_34" -LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto" - -# Apply installation instructions from official documentation: -# https://certbot.eff.org/lets-encrypt/centosrhel6-other -cp "$LE_AUTO" /usr/local/bin/certbot-auto -chown root /usr/local/bin/certbot-auto -chmod 0755 /usr/local/bin/certbot-auto -LE_AUTO=/usr/local/bin/certbot-auto - -# Last version of certbot-auto that was bootstraping Python 3.4 for CentOS 6 users -INITIAL_CERTBOT_VERSION_PY34="certbot 0.38.0" - -# Check bootstrap from current certbot-auto will fail, because SCL is not enabled. -set +o pipefail -if ! "$LE_AUTO" -n 2>&1 | grep -q "Enable the SCL repository and try running Certbot again."; then - echo "ERROR: Bootstrap was not aborted although SCL was not installed!" - exit 1 -fi -set -o pipefail - -echo "PASSED: Bootstrap was aborted since SCL was not installed." - -# Bootstrap from the old letsencrypt-auto, Python 3.4 will be installed from EPEL. -"$LE_AUTO_PY_34" --no-self-upgrade -n --install-only >/dev/null 2>/dev/null - -# Ensure Python 3.4 is installed -if ! command -v python3.4 &>/dev/null; then - echo "ERROR: old letsencrypt-auto failed to install Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present." - exit 1 -fi - -echo "PASSED: Bootstrap from old letsencrypt-auto succeeded and installed Python 3.4" - -# Expect certbot-auto to skip rebootstrapping with a warning since SCL is not installed. -if ! "$LE_AUTO" --non-interactive --version 2>&1 | grep -q "This requires manual user intervention"; then - echo "FAILED: Script certbot-auto did not print a warning about needing manual intervention!" - exit 1 -fi - -echo "PASSED: Script certbot-auto did not rebootstrap." - -# NB: Readline has an issue on all Python versions for OL 6, making `certbot --version` -# output an unprintable ASCII character on a new line at the end. -# So we take the second last line of the output. -version=$($LE_AUTO --version 2>/dev/null | tail -2 | head -1) - -if [ "$version" != "$INITIAL_CERTBOT_VERSION_PY34" ]; then - echo "ERROR: Script certbot-auto upgraded certbot in a non-interactive shell while SCL was not enabled." - exit 1 -fi - -echo "PASSED: Script certbot-auto did not upgrade certbot but started it successfully while SCL was not enabled." - -# Enable SCL -yum install -y oracle-softwarecollection-release-el6 >/dev/null - -# Expect certbot-auto to bootstrap successfully since SCL is available. -"$LE_AUTO" -n --version &>/dev/null - -if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1-2)" != "3.6" ]; then - echo "ERROR: Script certbot-auto failed to bootstrap and install Python 3.6 while SCL is available." - exit 1 -fi - -if ! /opt/eff.org/certbot/venv/bin/certbot --version > /dev/null 2> /dev/null; then - echo "ERROR: Script certbot-auto did not install certbot correctly while SCL is enabled." - exit 1 -fi - -echo "PASSED: Script certbot-auto correctly bootstraped Certbot using rh-python36 when SCL is available." - -# Expect certbot-auto will be totally silent now that everything has been correctly boostraped. -OUTPUT_LEN=$("$LE_AUTO" --install-only --no-self-upgrade --quiet 2>&1 | wc -c) -if [ "$OUTPUT_LEN" != 0 ]; then - echo certbot-auto produced unexpected output! - exit 1 -fi - -echo "PASSED: Script certbot-auto did not print anything in quiet mode." diff --git a/letsencrypt-auto-source/tests/uname_wrapper.sh b/letsencrypt-auto-source/tests/uname_wrapper.sh deleted file mode 100644 index df1f568c6..000000000 --- a/letsencrypt-auto-source/tests/uname_wrapper.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e - -uname_output=$(/bin/uname_orig "$@") - -if [ "$UNAME_FAKE_32BITS" = true ]; then - uname_output="${uname_output//x86_64/i686}" -fi - -echo "$uname_output" diff --git a/linter_plugin.py b/linter_plugin.py index 75879f73a..a19bf7df9 100644 --- a/linter_plugin.py +++ b/linter_plugin.py @@ -10,7 +10,9 @@ from pylint.checkers import BaseChecker from pylint.interfaces import IAstroidChecker # Modules in theses packages can import the os module. -WHITELIST_PACKAGES = ['acme', 'certbot_compatibility_test', 'lock_test'] +WHITELIST_PACKAGES = [ + 'acme', 'certbot_integration_tests', 'certbot_compatibility_test', 'lock_test' +] class ForbidStandardOsModule(BaseChecker): @@ -25,8 +27,8 @@ class ForbidStandardOsModule(BaseChecker): 'E5001': ( 'Forbidden use of os module, certbot.compat.os must be used instead', 'os-module-forbidden', - 'Some methods from the standard os module cannot be used for security reasons on Windows: ' - 'the safe wrapper certbot.compat.os must be used instead in Certbot.' + 'Some methods from the standard os module cannot be used for security reasons on ' + 'Windows: the safe wrapper certbot.compat.os must be used instead in Certbot.' ) } priority = -1 diff --git a/pull_request_template.md b/pull_request_template.md index c806d33e8..53298291b 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -1,4 +1,5 @@ ## Pull Request Checklist - [ ] If the change being made is to a [distributed component](https://certbot.eff.org/docs/contributing.html#code-components-and-layout), edit the `master` section of `certbot/CHANGELOG.md` to include a description of the change being made. +- [ ] Add or update any documentation as needed to support the changes in this PR. - [ ] Include your name in `AUTHORS.md` if you like. diff --git a/pytest.ini b/pytest.ini index 16aa9a193..b7a6928ea 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,6 +4,8 @@ [pytest] # In general, all warnings are treated as errors. Here are the exceptions: # 1- decodestring: https://github.com/rthalley/dnspython/issues/338 +# 2- Python 2 deprecation: https://github.com/certbot/certbot/issues/8388 +# (to be removed with Certbot 1.12.0 and its drop of Python 2 support) # Warnings being triggered by our plugins using deprecated features in # acme/certbot should be fixed by having our plugins no longer using the # deprecated code rather than adding them to the list of ignored warnings here. @@ -14,3 +16,4 @@ filterwarnings = error ignore:decodestring:DeprecationWarning + ignore:Python 2 support will be dropped:PendingDeprecationWarning diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 5fbf8503d..c9061ecb3 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -20,13 +20,13 @@ adopt-info: certbot apps: certbot: - command: bin/python3 $SNAP/bin/certbot + command: bin/python3 -s $SNAP/bin/certbot environment: PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" AUGEAS_LENS_LIB: "$SNAP/usr/share/augeas/lenses/dist" CERTBOT_SNAPPED: "True" renew: - command: bin/python3 $SNAP/bin/certbot -q renew + command: bin/python3 -s $SNAP/bin/certbot -q renew daemon: oneshot environment: PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" @@ -40,7 +40,6 @@ parts: certbot: plugin: python source: . - constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt] python-packages: - git+https://github.com/certbot/python-augeas.git@certbot-patched - ./acme @@ -64,7 +63,6 @@ parts: - libpython3.8-stdlib - libpython3.8-minimal - python3-pip - - python3-setuptools - python3-wheel - python3-venv - python3-minimal @@ -74,13 +72,23 @@ parts: # To build cryptography and cffi if needed build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev] build-environment: - - SNAPCRAFT_PYTHON_VENV_ARGS: --system-site-packages - - PIP_NO_BUILD_ISOLATION: "no" + - SNAPCRAFT_PYTHON_VENV_ARGS: --upgrade + # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the + # parts.[part_name].constraints option available in snapcraft.yaml when the Python plugin is + # used. This is done to let these constraints be applied not only on the certbot package + # build, but also on any isolated build that pip could trigger when building wheels for + # dependencies. See https://github.com/certbot/certbot/pull/8443 for more info. + - PIP_CONSTRAINT: $SNAPCRAFT_PART_SRC/snap-constraints.txt + override-build: | + python3 -m venv "${SNAPCRAFT_PART_INSTALL}" + "${SNAPCRAFT_PART_INSTALL}/bin/python3" "${SNAPCRAFT_PART_SRC}/tools/pipstrap.py" + snapcraftctl build override-pull: | - snapcraftctl pull - cd $SNAPCRAFT_PART_SRC - python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt | grep -v python-augeas > snap-constraints.txt - snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" $SNAPCRAFT_PART_SRC/certbot/certbot/__init__.py` + snapcraftctl pull + python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/letsencrypt-auto-source/pieces/dependency-requirements.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` shared-metadata: plugin: dump source: . diff --git a/tests/letstest/README.md b/tests/letstest/README.md index 4cf6c83c3..76db57153 100644 --- a/tests/letstest/README.md +++ b/tests/letstest/README.md @@ -1,7 +1,6 @@ # letstest Simple AWS testfarm scripts for certbot client testing -- Configures (canned) boulder server - Launches EC2 instances with a given list of AMIs for different distros - Copies certbot repo and puts it on the instances - Runs certbot tests (bash scripts) on all of these @@ -56,11 +55,6 @@ It will take a minute for these instances to shut down and become available agai A folder named `letest-` is also created with a log file from each instance of the test and a file named "results" containing the output above. The tests take quite a while to run. -Also, the way all of the tests work is to check if there is already a boulder server running and if not start one. The boulder server is left running between tests, -and there are known issues if two instances of boulder attempt to be started. After starting your first test, wait until you see "Found existing boulder server:" or if you see output -about creating a boulder server, wait a minute before starting the 2nd test. You only have to do this after starting your first session of tests or after running -the `aws ec2 terminate-instances` command above. - ## Scripts Example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed to them at runtime via environment variables. test_apache2.sh is a useful reference. @@ -73,5 +67,4 @@ See: - https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html Main repos: -- https://github.com/letsencrypt/boulder - https://github.com/letsencrypt/letsencrypt diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml index 8e8e23116..2663782ce 100644 --- a/tests/letstest/apache2_targets.yaml +++ b/tests/letstest/apache2_targets.yaml @@ -1,4 +1,7 @@ # These images are located in us-east-1. +# +# All machines must currently use x86_64 since Pebble does not currently +# publish images for other architectures. targets: #----------------------------------------------------------------------------- @@ -30,12 +33,6 @@ targets: type: ubuntu virt: hvm user: admin - - ami: ami-0dcd54b7d2fff584f - name: debian10_arm64 - type: ubuntu - virt: hvm - user: admin - machine_type: a1.medium - ami: ami-003f19e0e687de1cd name: debian9 type: ubuntu diff --git a/tests/letstest/auto_targets.yaml b/tests/letstest/auto_targets.yaml index 76b3a3dc5..01d410227 100644 --- a/tests/letstest/auto_targets.yaml +++ b/tests/letstest/auto_targets.yaml @@ -31,10 +31,6 @@ targets: virt: hvm user: admin machine_type: a1.medium - # userdata: | - # #cloud-init - # runcmd: - # - [ apt-get, install, -y, curl ] #----------------------------------------------------------------------------- # Other Redhat Distros - ami: ami-0916c408cb02e310b @@ -56,17 +52,6 @@ targets: type: centos virt: hvm user: centos - # centos6 requires EPEL repo added - - ami: ami-1585c46a - name: centos6 - type: centos - virt: hvm - user: centos - userdata: | - #cloud-config - runcmd: - - yum install -y epel-release - - iptables -F - ami: ami-01ca03df4a6012157 name: centos8 type: centos diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index cf9f2899a..5ad1d8c15 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -1,7 +1,6 @@ """ Certbot Integration Test Tool -- Configures (canned) boulder server - Launches EC2 instances with a given list of AMIs for different distros - Copies certbot repo and puts it on the instances - Runs certbot tests (bash scripts) on all of these @@ -81,12 +80,6 @@ parser.add_argument('--saveinstances', parser.add_argument('--alt_pip', default='', help="server from which to pull candidate release packages") -parser.add_argument('--killboulder', - action='store_true', - help="do not leave a persistent boulder server running") -parser.add_argument('--boulderonly', - action='store_true', - help="only make a boulder server") cl_args = parser.parse_args() # Credential Variables @@ -98,7 +91,6 @@ PROFILE = None if cl_args.aws_profile == 'SET_BY_ENV' else cl_args.aws_profile # Globals #------------------------------------------------------------------------------- -BOULDER_AMI = 'ami-072a9534772bec854' # premade shared boulder AMI 18.04LTS us-east-1 SECURITY_GROUP_NAME = 'certbot-security-group' SENTINEL = None #queue kill signal SUBNET_NAME = 'certbot-subnet' @@ -133,10 +125,6 @@ def make_security_group(vpc): mysg = vpc.create_security_group(GroupName=SECURITY_GROUP_NAME, Description='security group for automated testing') mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22) - mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80) - mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=443, ToPort=443) - # for boulder wfe (http) server - mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=4000, ToPort=4000) # for mosh mysg.authorize_ingress(IpProtocol="udp", CidrIp="0.0.0.0/0", FromPort=60000, ToPort=61000) return mysg @@ -147,22 +135,32 @@ def make_instance(ec2_client, keyname, security_group_id, subnet_id, - machine_type='t2.micro', - userdata=""): #userdata contains bash or cloud-init script + self_destruct, + machine_type='t2.micro'): + """Creates an instance using the given parameters. + + If self_destruct is True, the instance will be configured to shutdown after + 1 hour and to terminate itself on shutdown. + + """ block_device_mappings = _get_block_device_mappings(ec2_client, ami_id) tags = [{'Key': 'Name', 'Value': instance_name}] tag_spec = [{'ResourceType': 'instance', 'Tags': tags}] - return ec2_client.create_instances( - BlockDeviceMappings=block_device_mappings, - ImageId=ami_id, - SecurityGroupIds=[security_group_id], - SubnetId=subnet_id, - KeyName=keyname, - MinCount=1, - MaxCount=1, - UserData=userdata, - InstanceType=machine_type, - TagSpecifications=tag_spec)[0] + kwargs = { + 'BlockDeviceMappings': block_device_mappings, + 'ImageId': ami_id, + 'SecurityGroupIds': [security_group_id], + 'SubnetId': subnet_id, + 'KeyName': keyname, + 'MinCount': 1, + 'MaxCount': 1, + 'InstanceType': machine_type, + 'TagSpecifications': tag_spec + } + if self_destruct: + kwargs['InstanceInitiatedShutdownBehavior'] = 'terminate' + kwargs['UserData'] = '#!/bin/bash\nshutdown -P +60\n' + return ec2_client.create_instances(**kwargs)[0] def _get_block_device_mappings(ec2_client, ami_id): """Returns the list of block device mappings to ensure cleanup. @@ -183,23 +181,6 @@ def _get_block_device_mappings(ec2_client, ami_id): # Helper Routines #------------------------------------------------------------------------------- -def block_until_http_ready(urlstring, wait_time=10, timeout=240): - "Blocks until server at urlstring can respond to http requests" - server_ready = False - t_elapsed = 0 - while not server_ready and t_elapsed < timeout: - try: - sys.stdout.write('.') - sys.stdout.flush() - req = urllib_request.Request(urlstring) - response = urllib_request.urlopen(req) - #if response.code == 200: - server_ready = True - except urllib_error.URLError: - pass - time.sleep(wait_time) - t_elapsed += wait_time - def block_until_ssh_open(ipstring, wait_time=10, timeout=120): "Blocks until server at ipstring has an open port 22" reached = False @@ -278,26 +259,15 @@ def deploy_script(cxn, scriptpath, *args): args_str = ' '.join(args) cxn.run('./'+scriptfile+' '+args_str) -def run_boulder(cxn): - boulder_path = '$GOPATH/src/github.com/letsencrypt/boulder' - cxn.run('cd %s && sudo docker-compose up -d' % boulder_path) - -def config_and_launch_boulder(cxn, instance): - # yes, we're hardcoding the gopath. it's a predetermined AMI. - with cxn.prefix('export GOPATH=/home/ubuntu/gopath'): - deploy_script(cxn, 'scripts/boulder_config.sh') - run_boulder(cxn) - -def install_and_launch_certbot(cxn, instance, boulder_url, target, log_dir): +def install_and_launch_certbot(cxn, instance, target, log_dir): local_repo_to_remote(cxn, log_dir) # This needs to be like this, I promise. 1) The env argument to run doesn't work. # See https://github.com/fabric/fabric/issues/1744. 2) prefix() sticks an && between # the commands, so it needs to be exports rather than no &&s in between for the script subshell. - with cxn.prefix('export BOULDER_URL=%s && export PUBLIC_IP=%s && export PRIVATE_IP=%s && ' + with cxn.prefix('export PUBLIC_IP=%s && export PRIVATE_IP=%s && ' 'export PUBLIC_HOSTNAME=%s && export PIP_EXTRA_INDEX_URL=%s && ' 'export OS_TYPE=%s' % - (boulder_url, - instance.public_ip_address, + (instance.public_ip_address, instance.private_ip_address, instance.public_dns_name, cl_args.alt_pip, @@ -313,7 +283,7 @@ def grab_certbot_log(cxn): 'cat ./certbot.log; else echo "[nolocallog]"; fi\'') -def create_client_instance(ec2_client, target, security_group_id, subnet_id): +def create_client_instance(ec2_client, target, security_group_id, subnet_id, self_destruct): """Create a single client instance for running tests.""" if 'machine_type' in target: machine_type = target['machine_type'] @@ -322,10 +292,6 @@ def create_client_instance(ec2_client, target, security_group_id, subnet_id): else: # 32 bit systems machine_type = 'c1.medium' - if 'userdata' in target: - userdata = target['userdata'] - else: - userdata = '' name = 'le-%s'%target['name'] print(name, end=" ") return make_instance(ec2_client, @@ -335,10 +301,10 @@ def create_client_instance(ec2_client, target, security_group_id, subnet_id): machine_type=machine_type, security_group_id=security_group_id, subnet_id=subnet_id, - userdata=userdata) + self_destruct=self_destruct) -def test_client_process(fab_config, inqueue, outqueue, boulder_url, log_dir): +def test_client_process(fab_config, inqueue, outqueue, log_dir): cur_proc = mp.current_process() for inreq in iter(inqueue.get, SENTINEL): ii, instance_id, target = inreq @@ -360,7 +326,7 @@ def test_client_process(fab_config, inqueue, outqueue, boulder_url, log_dir): with Connection(host_string, config=fab_config) as cxn: try: - install_and_launch_certbot(cxn, instance, boulder_url, target, log_dir) + install_and_launch_certbot(cxn, instance, target, log_dir) outqueue.put((ii, target, Status.PASS)) print("%s - %s SUCCESS"%(target['ami'], target['name'])) except: @@ -379,15 +345,13 @@ def test_client_process(fab_config, inqueue, outqueue, boulder_url, log_dir): pass -def cleanup(cl_args, instances, targetlist, boulder_server, log_dir): +def cleanup(cl_args, instances, targetlist, log_dir): print('Logs in ', log_dir) # If lengths of instances and targetlist aren't equal, instances failed to # start before running tests so leaving instances running for debugging # isn't very useful. Let's cleanup after ourselves instead. if len(instances) != len(targetlist) or not cl_args.saveinstances: print('Terminating EC2 Instances') - if cl_args.killboulder: - boulder_server.terminate() for instance in instances: instance.terminate() else: @@ -477,63 +441,18 @@ def main(): security_group_id = make_security_group(vpc).id time.sleep(30) - boulder_preexists = False - boulder_servers = ec2_client.instances.filter(Filters=[ - {'Name': 'tag:Name', 'Values': ['le-boulderserver']}, - {'Name': 'instance-state-name', 'Values': ['running']}]) - - boulder_server = next(iter(boulder_servers), None) - - print("Requesting Instances...") - if boulder_server: - print("Found existing boulder server:", boulder_server) - boulder_preexists = True - else: - print("Can't find a boulder server, starting one...") - boulder_server = make_instance(ec2_client, - 'le-boulderserver', - BOULDER_AMI, - KEYNAME, - machine_type='t2.micro', - #machine_type='t2.medium', - security_group_id=security_group_id, - subnet_id=subnet_id) - instances = [] try: - if not cl_args.boulderonly: - print("Creating instances: ", end="") - for target in targetlist: - instances.append( - create_client_instance(ec2_client, target, - security_group_id, subnet_id) - ) - print() - - # Configure and launch boulder server - #------------------------------------------------------------------------------- - print("Waiting on Boulder Server") - boulder_server = block_until_instance_ready(boulder_server) - print(" server %s"%boulder_server) - - - # host_string defines the ssh user and host for connection - host_string = "ubuntu@%s"%boulder_server.public_ip_address - print("Boulder Server at (SSH):", host_string) - if not boulder_preexists: - print("Configuring and Launching Boulder") - with Connection(host_string, config=fab_config) as boulder_cxn: - config_and_launch_boulder(boulder_cxn, boulder_server) - # blocking often unnecessary, but cheap EC2 VMs can get very slow - block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, - wait_time=10, timeout=500) - - boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address - print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) - print("Boulder Server at (EC2 private ip): %s"%boulder_url) - - if cl_args.boulderonly: - sys.exit(0) + print("Creating instances: ", end="") + # If we want to preserve instances, do not have them self-destruct. + self_destruct = not cl_args.saveinstances + for target in targetlist: + instances.append( + create_client_instance(ec2_client, target, + security_group_id, subnet_id, + self_destruct) + ) + print() # Install and launch client scripts in parallel #------------------------------------------------------------------------------- @@ -551,7 +470,7 @@ def main(): # initiate process execution - client_process_args=(fab_config, inqueue, outqueue, boulder_url, log_dir) + client_process_args=(fab_config, inqueue, outqueue, log_dir) for i in range(num_processes): p = mp.Process(target=test_client_process, args=client_process_args) jobs.append(p) @@ -602,7 +521,7 @@ def main(): sys.exit(1) finally: - cleanup(cl_args, instances, targetlist, boulder_server, log_dir) + cleanup(cl_args, instances, targetlist, log_dir) if __name__ == '__main__': diff --git a/tests/letstest/scripts/bootstrap_os_packages.sh b/tests/letstest/scripts/bootstrap_os_packages.sh index 96506282b..7ad93f63e 100755 --- a/tests/letstest/scripts/bootstrap_os_packages.sh +++ b/tests/letstest/scripts/bootstrap_os_packages.sh @@ -98,41 +98,6 @@ BootstrapRpmCommonBase() { fi } -# This bootstrap concerns old RedHat-based distributions that do not ship by default -# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing -# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6. -BootstrapRpmPython3Legacy() { - # Tested with: - # - CentOS 6 - - InitializeRPMCommonBase - - if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then - echo "To use Certbot on this operating system, packages from the SCL repository need to be installed." - if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then - error "Enable the SCL repository and try running Certbot again." - exit 1 - fi - if ! "${TOOL}" install -y centos-release-scl; then - error "Could not enable SCL. Aborting bootstrap!" - exit 1 - fi - fi - - # CentOS 6 must use rh-python36 from SCL - if "${TOOL}" list rh-python36 >/dev/null 2>&1; then - python_pkgs="rh-python36-python - rh-python36-python-virtualenv - rh-python36-python-devel - " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 - fi - - BootstrapRpmCommonBase "${python_pkgs}" -} - BootstrapRpmPython3() { InitializeRPMCommonBase @@ -154,16 +119,9 @@ if [ -f /etc/debian_version ]; then } elif [ -f /etc/redhat-release ]; then DeterminePythonVersion - # Handle legacy RPM distributions - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapRpmPython3Legacy - } - else - Bootstrap() { - BootstrapRpmPython3 - } - fi + Bootstrap() { + BootstrapRpmPython3 + } fi diff --git a/tests/letstest/scripts/boulder_config.sh b/tests/letstest/scripts/boulder_config.sh deleted file mode 100755 index b99bbabbe..000000000 --- a/tests/letstest/scripts/boulder_config.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -x - -# Configures and Launches Boulder Server installed on -# us-east-1 ami-072a9534772bec854 bouldertestserver3 (boulder commit b24fe7c3ea4) - -# fetch instance data from EC2 metadata service -public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) -public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) -private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) - -# set to public DNS resolver -resolver_ip=8.8.8.8 -resolver=$resolver_ip':53' - -# modifies integration testing boulder setup for local AWS VPC network -# connections instead of localhost -cd $GOPATH/src/github.com/letsencrypt/boulder -# change test ports to real -sed -i '/httpPort/ s/5002/80/' ./test/config/va.json -sed -i '/httpsPort/ s/5001/443/' ./test/config/va.json -sed -i '/tlsPort/ s/5001/443/' ./test/config/va.json -# set dns resolver -sed -i 's/"127.0.0.1:8053",/"'$resolver'"/' ./test/config/va.json -sed -i 's/"127.0.0.1:8054"//' ./test/config/va.json diff --git a/tests/letstest/scripts/boulder_install.sh b/tests/letstest/scripts/boulder_install.sh deleted file mode 100755 index 5161de374..000000000 --- a/tests/letstest/scripts/boulder_install.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -x - -# Check out special branch until latest docker changes land in Boulder master. -git clone -b docker-integration https://github.com/letsencrypt/boulder $BOULDERPATH -cd $BOULDERPATH -FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') -sed -i "s/FAKE_DNS: .*/FAKE_DNS: $FAKE_DNS/" docker-compose.yml -docker-compose up -d diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index ba3d94379..247191610 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -7,7 +7,7 @@ if [ "$OS_TYPE" = "ubuntu" ] then CONFFILE=/etc/apache2/sites-available/000-default.conf sudo apt-get update - sudo apt-get -y --no-upgrade install apache2 #curl + sudo apt-get -y --no-upgrade install apache2 curl sudo apt-get -y install realpath # needed for test-apache-conf # For apache 2.4, set up ServerName sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE @@ -64,17 +64,41 @@ if [ $? -ne 0 ] ; then exit 1 fi -if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then - # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment - # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap - PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" +tools/venv3.py -e acme[dev] -e certbot[dev,docs] -e certbot-apache -e certbot-ci +PEBBLE_LOGS="acme_server.log" +PEBBLE_URL="https://localhost:14000/dir" +# We configure Pebble to use port 80 for http-01 validation rather than an +# alternate port because: +# 1) It allows us to test with Apache configurations that are more realistic +# and closer to the default configuration on various OSes. +# 2) As of writing this, Certbot's Apache plugin requires there to be an +# existing virtual host for the port used for http-01 validation. +venv3/bin/run_acme_server --http-01-port 80 > "${PEBBLE_LOGS}" 2>&1 & + +DumpPebbleLogs() { + if [ -f "${PEBBLE_LOGS}" ] ; then + echo "Pebble's logs were:" + cat "${PEBBLE_LOGS}" + fi +} + +for n in $(seq 1 150) ; do + if curl --insecure "${PEBBLE_URL}" 2>/dev/null; then + break + else + echo "waiting for pebble" + sleep 1 + fi +done +if ! curl --insecure "${PEBBLE_URL}" 2>/dev/null; then + echo "timed out waiting for pebble to start" + DumpPebbleLogs + exit 1 fi -tools/venv3.py -e acme[dev] -e certbot[dev,docs] -e certbot-apache - -sudo "venv3/bin/certbot" -v --debug --text --agree-tos \ +sudo "venv3/bin/certbot" -v --debug --text --agree-tos --no-verify-ssl \ --renew-by-default --redirect --register-unsafely-without-email \ - --domain $PUBLIC_HOSTNAME --server $BOULDER_URL + --domain "${PUBLIC_HOSTNAME}" --server "${PEBBLE_URL}" if [ $? -ne 0 ] ; then FAIL=1 fi @@ -96,7 +120,7 @@ fi if [ "$OS_TYPE" = "ubuntu" ] ; then - export SERVER="$BOULDER_URL" + export SERVER="${PEBBLE_URL}" "venv3/bin/tox" -e apacheconftest else echo Not running hackish apache tests on $OS_TYPE @@ -108,5 +132,6 @@ fi # return error if any of the subtests failed if [ "$FAIL" = 1 ] ; then + DumpPebbleLogs exit 1 fi diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 51ff640c5..c599623cb 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -1,7 +1,7 @@ #!/bin/bash -xe set -o pipefail -# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL +# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME # are dynamically set at execution cd letsencrypt @@ -105,15 +105,10 @@ if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python exit 1 fi -# On systems like Debian where certbot-auto is deprecated, we expect it to -# leave existing Certbot installations unmodified so we check for the same -# version that was initially installed below. Once certbot-auto is deprecated -# on RHEL systems, we can unconditionally check for INITIAL_VERSION. -if [ -f /etc/debian_version ]; then - EXPECTED_VERSION="$INITIAL_VERSION" -else - EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) -fi +# Since certbot-auto is deprecated, we expect it to leave existing Certbot +# installations unmodified so we check for the same version that was initially +# installed below. +EXPECTED_VERSION="$INITIAL_VERSION" if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then echo unexpected certbot version found @@ -124,22 +119,3 @@ if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ exit 1 fi - -if [ "$RUN_RHEL6_TESTS" = 1 ]; then - # Add the SCL python release to PATH in order to resolve python3 command - PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" - if ! command -v python3; then - echo "Python3 wasn't properly installed" - exit 1 - fi - if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1)" != 3 ]; then - echo "Python3 wasn't used in venv!" - exit 1 - fi - - if [ "$("$PYTHON_NAME" tools/readlink.py $OLD_VENV_PATH)" != "/opt/eff.org/certbot/venv" ]; then - echo symlink from old venv path not properly created! - exit 1 - fi -fi -echo upgrade appeared to be successful diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh index 15cf9ee1b..9573ab690 100755 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh @@ -1,7 +1,7 @@ #!/bin/bash -x set -eo pipefail -# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution +# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME are dynamically set at execution # with curl, instance metadata available from EC2 metadata service: #public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) @@ -16,58 +16,14 @@ sudo chown root "$LE_AUTO_PATH" sudo chmod 0755 "$LE_AUTO_PATH" export PATH="$LE_AUTO_DIR:$PATH" -# On systems like Debian where certbot-auto is deprecated, we expect -# certbot-auto to error and refuse to install Certbot. Once certbot-auto is -# deprecated on RHEL systems, we can unconditionally run this code. -if [ -f /etc/debian_version ]; then - set +o pipefail - if ! letsencrypt-auto --debug --version | grep "Certbot cannot be installed."; then - echo "letsencrypt-auto didn't report being uninstallable." - exit 1 - fi - if [ ${PIPESTATUS[0]} != 1 ]; then - echo "letsencrypt-auto didn't exit with status 1 as expected" - exit 1 - fi - # letsencrypt-auto is deprecated and cannot be installed on this system so - # we cannot run the rest of this test. - exit 0 -fi - -letsencrypt-auto --os-packages-only --debug --version - -# This script sets the environment variables PYTHON_NAME, VENV_PATH, and -# VENV_SCRIPT based on the version of Python available on the system. For -# instance, Fedora uses Python 3 and Python 2 is not installed. -. tests/letstest/scripts/set_python_envvars.sh - -# Create a venv-like layout at the old virtual environment path to test that a -# symlink is properly created when letsencrypt-auto runs. -HOME=${HOME:-~root} -XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} -OLD_VENV_BIN="$XDG_DATA_HOME/letsencrypt/bin" -mkdir -p "$OLD_VENV_BIN" -touch "$OLD_VENV_BIN/letsencrypt" - -letsencrypt-auto certonly --no-self-upgrade -v --standalone --debug \ - --text --agree-tos \ - --renew-by-default --redirect \ - --register-unsafely-without-email \ - --domain $PUBLIC_HOSTNAME --server $BOULDER_URL - -LINK_PATH=$("$PYTHON_NAME" tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt) -if [ "$LINK_PATH" != "/opt/eff.org/certbot/venv" ]; then - echo symlink from old venv path not properly created! +# Since certbot-auto is deprecated, we expect certbot-auto to error and +# refuse to install Certbot. +set +o pipefail +if ! letsencrypt-auto --debug --version | grep "Certbot cannot be installed."; then + echo "letsencrypt-auto didn't report being uninstallable." exit 1 fi - -if ! letsencrypt-auto --help --no-self-upgrade | grep -F "letsencrypt-auto [SUBCOMMAND]"; then - echo "letsencrypt-auto not included in help output!" - exit 1 -fi - -OUTPUT_LEN=$(letsencrypt-auto --install-only --no-self-upgrade --quiet 2>&1 | wc -c) -if [ "$OUTPUT_LEN" != 0 ]; then - echo letsencrypt-auto produced unexpected output! +if [ ${PIPESTATUS[0]} != 1 ]; then + echo "letsencrypt-auto didn't exit with status 1 as expected" exit 1 fi diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index e3d9a8b80..a038caff6 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -8,12 +8,6 @@ VENV_PATH=venv3 # install OS packages sudo $BOOTSTRAP_SCRIPT -if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then - # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment - # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap - PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" -fi - # setup venv # We strip the hashes because the venv creation script includes unhashed # constraints in the commands given to pip and the mix of hashed and unhashed diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index f62584709..f07e3b78e 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -15,12 +15,6 @@ VENV_SCRIPT="tools/venv3.py" sudo $BOOTSTRAP_SCRIPT -if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then - # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment - # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap - PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" -fi - cd $REPO_ROOT $VENV_SCRIPT . $VENV_NAME/bin/activate diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 29edd1552..97c775f6c 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -52,17 +52,6 @@ targets: type: centos virt: hvm user: centos - # centos6 requires EPEL repo added - - ami: ami-1585c46a - name: centos6 - type: centos - virt: hvm - user: centos - userdata: | - #cloud-config - runcmd: - - yum install -y epel-release - - iptables -F - ami: ami-01ca03df4a6012157 name: centos8 type: centos diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index 02222008b..0d3626853 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -42,9 +42,7 @@ RUN apk add --no-cache --virtual .build-deps \ musl-dev \ libffi-dev \ && python tools/pipstrap.py \ - && pip install --no-build-isolation \ - -r letsencrypt-auto-source/pieces/dependency-requirements.txt \ - && pip install --no-build-isolation --no-cache-dir --no-deps \ - --editable src/acme \ - --editable src/certbot \ -&& apk del .build-deps + && python tools/pip_install.py --no-cache-dir \ + --editable src/acme \ + --editable src/certbot \ + && apk del .build-deps diff --git a/tools/docker/plugin/Dockerfile b/tools/docker/plugin/Dockerfile index 5a6673e5b..863efd105 100644 --- a/tools/docker/plugin/Dockerfile +++ b/tools/docker/plugin/Dockerfile @@ -11,4 +11,4 @@ COPY qemu-${QEMU_ARCH}-static /usr/bin/ COPY . /opt/certbot/src/plugin # Install the DNS plugin -RUN tools/pip_install.py --no-cache-dir --editable /opt/certbot/src/plugin +RUN python tools/pip_install.py --no-cache-dir --editable /opt/certbot/src/plugin diff --git a/tools/finish_release.py b/tools/finish_release.py index bc8e832df..24e20987f 100755 --- a/tools/finish_release.py +++ b/tools/finish_release.py @@ -21,6 +21,7 @@ Run: python tools/finish_release.py ~/.ssh/githubpat.txt """ +import argparse import glob import os.path import re @@ -44,6 +45,34 @@ SNAPS = ['certbot'] + DNS_PLUGINS # for sanity checking. SNAP_ARCH_COUNT = 3 + +def parse_args(args): + """Parse command line arguments. + + :param args: command line arguments with the program name removed. This is + usually taken from sys.argv[1:]. + :type args: `list` of `str` + + :returns: parsed arguments + :rtype: argparse.Namespace + + """ + # Use the file's docstring for the help text and don't let argparse reformat it. + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('githubpat', help='path to your GitHub personal access token') + group = parser.add_mutually_exclusive_group() + # We use 'store_false' and a destination related to the other type of + # artifact to cause the flag being set to disable publishing of the other + # artifact. This makes using the parsed arguments later on a little simpler + # and cleaner. + group.add_argument('--snaps-only', action='store_false', dest='publish_windows', + help='Skip publishing other artifacts and only publish the snaps') + group.add_argument('--windows-only', action='store_false', dest='publish_snaps', + help='Skip publishing other artifacts and only publish the Windows installer') + return parser.parse_args(args) + + def download_azure_artifacts(tempdir): """Download and unzip build artifacts from Azure pipelines. @@ -181,8 +210,9 @@ def promote_snaps(version): def main(args): - github_access_token_file = args[0] + parsed_args = parse_args(args) + github_access_token_file = parsed_args.githubpat github_access_token = open(github_access_token_file, 'r').read().rstrip() with tempfile.TemporaryDirectory() as tempdir: @@ -191,8 +221,10 @@ def main(args): # again fails. Publishing the snaps can be done multiple times though # so we do that first to make it easier to run the script again later # if something goes wrong. - promote_snaps(version) - create_github_release(github_access_token, tempdir, version) + if parsed_args.publish_snaps: + promote_snaps(version) + if parsed_args.publish_windows: + create_github_release(github_access_token, tempdir, version) if __name__ == "__main__": main(sys.argv[1:]) diff --git a/tools/pip_install.py b/tools/pip_install.py index f963e4660..c1c81482b 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -59,9 +59,13 @@ def certbot_normal_processing(tools_path, test_constraints): certbot_requirements = os.path.normpath(os.path.join( repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) with open(certbot_requirements, 'r') as fd: - data = fd.readlines() + certbot_reqs = fd.readlines() + with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd: + pipstrap_reqs = fd.readlines() with open(test_constraints, 'w') as fd: - data = "\n".join(strip_hashes.process_entries(data)) + data_certbot = "\n".join(strip_hashes.process_entries(certbot_reqs)) + data_pipstrap = "\n".join(strip_hashes.process_entries(pipstrap_reqs)) + data = "\n".join([data_certbot, data_pipstrap]) fd.write(data) @@ -72,7 +76,8 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain # Here is the order by increasing priority: # 1) The general development constraints (tools/dev_constraints.txt) # 2) The general tests constraints (oldest_requirements.txt or - # certbot-auto's dependency-requirements.txt for the normal processing) + # certbot-auto's dependency-requirements.txt + pipstrap's constraints + # for the normal processing) # 3) The local requirement file, typically local-oldest-requirement in oldest tests files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] if requirements: @@ -82,17 +87,18 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain fd.write(merged_requirements) -def call_with_print(command): +def call_with_print(command, env=None): + if not env: + env = os.environ print(command) - subprocess.check_call(command, shell=True) + subprocess.check_call(command, shell=True, env=env) -def pip_install_with_print(args_str, disable_build_isolation=True): - command = ['"', sys.executable, '" -m pip install --disable-pip-version-check '] - if disable_build_isolation: - command.append('--no-build-isolation ') - command.append(args_str) - call_with_print(''.join(command)) +def pip_install_with_print(args_str, env=None): + if not env: + env = os.environ + command = ['"', sys.executable, '" -m pip install --disable-pip-version-check ', args_str] + call_with_print(''.join(command), env=env) def main(args): @@ -113,20 +119,22 @@ def main(args): else: certbot_normal_processing(tools_path, test_constraints) + env = os.environ.copy() + env["PIP_CONSTRAINT"] = all_constraints + merge_requirements(tools_path, requirements, test_constraints, all_constraints) if requirements: # This branch is executed during the oldest tests # First step, install the transitive dependencies of oldest requirements # in respect with oldest constraints. - pip_install_with_print('--constraint "{0}" --requirement "{1}"' - .format(all_constraints, requirements)) + pip_install_with_print('--requirement "{0}"'.format(requirements), + env=env) # Second step, ensure that oldest requirements themselves are effectively # installed using --force-reinstall, and avoid corner cases like the one described # in https://github.com/certbot/certbot/issues/7014. pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"' .format(requirements)) - pip_install_with_print('--constraint "{0}" {1}'.format( - all_constraints, ' '.join(args))) + pip_install_with_print(' '.join(args), env=env) if __name__ == '__main__': diff --git a/tools/pipstrap.py b/tools/pipstrap.py index 2f21a9a5f..e6b746916 100755 --- a/tools/pipstrap.py +++ b/tools/pipstrap.py @@ -1,46 +1,17 @@ #!/usr/bin/env python """Uses pip to upgrade Python packaging tools to pinned versions.""" from __future__ import absolute_import - import os -import shutil -import tempfile import pip_install -# We include the hashes of the packages here for extra verification of -# the packages downloaded from PyPI. This is especially valuable in our -# builds of Certbot that we ship to our users such as our Docker images. -# -# An older version of setuptools is currently used here in order to keep -# compatibility with Python 2 since newer versions of setuptools have dropped -# support for it. -REQUIREMENTS = r""" -pip==20.2.4 \ - --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \ - --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1 -setuptools==44.1.1 \ - --hash=sha256:27a714c09253134e60a6fa68130f78c7037e5562c4f21f8f318f2ae900d152d5 \ - --hash=sha256:c67aa55db532a0dadc4d2e20ba9961cbd3ccc84d544e9029699822542b5a476b -wheel==0.35.1 \ - --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \ - --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f -""" +_REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_constraints.txt") def main(): - with pip_install.temporary_directory() as tempdir: - requirements_filepath = os.path.join(tempdir, 'reqs.txt') - with open(requirements_filepath, 'w') as f: - f.write(REQUIREMENTS) - pip_install_args = '--requirement ' + requirements_filepath - # We don't disable build isolation because we may have an older - # version of pip that doesn't support the flag disabling it. We - # expect these packages to already have usable wheels available - # anyway so no building should be required. - pip_install.pip_install_with_print(pip_install_args, - disable_build_isolation=False) + pip_install_args = '--requirement "{0}"'.format(_REQUIREMENTS_PATH) + pip_install.pip_install_with_print(pip_install_args) if __name__ == '__main__': diff --git a/tools/pipstrap_constraints.txt b/tools/pipstrap_constraints.txt new file mode 100644 index 000000000..5de9e147d --- /dev/null +++ b/tools/pipstrap_constraints.txt @@ -0,0 +1,18 @@ +# Constraints for pipstrap.py +# +# We include the hashes of the packages here for extra verification of +# the packages downloaded from PyPI. This is especially valuable in our +# builds of Certbot that we ship to our users such as our Docker images. +# +# An older version of setuptools is currently used here in order to keep +# compatibility with Python 2 since newer versions of setuptools have dropped +# support for it. +pip==20.2.4 \ + --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \ + --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1 +setuptools==44.1.1 \ + --hash=sha256:27a714c09253134e60a6fa68130f78c7037e5562c4f21f8f318f2ae900d152d5 \ + --hash=sha256:c67aa55db532a0dadc4d2e20ba9961cbd3ccc84d544e9029699822542b5a476b +wheel==0.35.1 \ + --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \ + --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index 285521190..e6a44240f 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -1,22 +1,22 @@ #!/usr/bin/env python3 import argparse -import glob import datetime -from multiprocessing import Pool, Process, Manager, Event +import glob import re import subprocess import sys -import tempfile +import time +from multiprocessing import Pool, Process, Manager from os.path import join, realpath, dirname, basename, exists - CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] def _execute_build(target, archs, status, workspace): process = subprocess.Popen([ - 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', '--build-on', ','.join(archs) + 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', + '--build-on', ','.join(archs) ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, cwd=workspace) process_output = [] @@ -27,7 +27,7 @@ def _execute_build(target, archs, status, workspace): return process.wait(), process_output -def _build_snap(target, archs, status, lock): +def _build_snap(target, archs, status, running, lock): status[target] = {arch: '...' for arch in archs} if target == 'certbot': @@ -39,7 +39,8 @@ def _build_snap(target, archs, status, lock): while retry: exit_code, process_output = _execute_build(target, archs, status, workspace) - print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with exit code {exit_code}.') + print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with ' + f'exit code {exit_code}.') sys.stdout.flush() with lock: @@ -49,7 +50,8 @@ def _build_snap(target, archs, status, lock): # We expect to have all target snaps available, or something bad happened. snaps_list = glob.glob(join(workspace, '*.snap')) if not len(snaps_list) == len(archs): - print(f'Some of the expected snaps for a successful build are missing (current list: {snaps_list}).') + print('Some of the expected snaps for a successful build are missing ' + f'(current list: {snaps_list}).') dump_output = True else: break @@ -63,9 +65,12 @@ def _build_snap(target, archs, status, lock): print(f'Dumping snapcraft remote-build output build for {target}:') print('\n'.join(process_output)) - # Retry the remote build if it has been interrupted (non zero status code) or if some builds have failed. + # Retry the remote build if it has been interrupted (non zero status code) + # or if some builds have failed. retry = retry - 1 + running[target] = False + return {target: workspace} @@ -96,15 +101,11 @@ def _dump_status_helper(archs, status): sys.stdout.flush() -def _dump_status(archs, status, stop_event): - while not stop_event.wait(10): - print('Remote build status at {0}'.format(datetime.datetime.now())) +def _dump_status(archs, status, running): + while any(running.values()): + print(f'Remote build status at {datetime.datetime.now()}') _dump_status_helper(archs, status) - - -def _dump_status_final(archs, status): - print('Results for remote build finished at {0}'.format(datetime.datetime.now())) - _dump_status_helper(archs, status) + time.sleep(10) def _dump_results(targets, archs, status, workspaces): @@ -120,10 +121,10 @@ def _dump_results(targets, archs, status, workspaces): if not exists(build_output_path): build_output = f'No output has been dumped by snapcraft remote-build.' else: - with open(join(workspaces[target], '{0}_{1}.txt'.format(target, arch))) as file_h: + with open(join(workspaces[target], f'{target}_{arch}.txt')) as file_h: build_output = file_h.read() - print('Output for failed build target={0} arch={1}'.format(target, arch)) + print(f'Output for failed build target={target} arch={arch}') print('-------------------------------------------') print(build_output) print('-------------------------------------------') @@ -134,6 +135,10 @@ def _dump_results(targets, archs, status, workspaces): else: print('Some builds failed.') + print() + print(f'Results for remote build finished at {datetime.datetime.now()}') + _dump_status_helper(archs, status) + return failures @@ -143,6 +148,8 @@ def main(): help='the list of snaps to build') parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default=['amd64'], help='the architectures for which snaps are built') + parser.add_argument('--timeout', type=int, default=None, + help='build process will fail after the provided timeout (in seconds)') args = parser.parse_args() archs = set(args.archs) @@ -158,7 +165,7 @@ def main(): # If we're building anything other than just Certbot, we need to # generate the snapcraft files for the DNS plugins. - if targets != set(('certbot',)): + if targets != {'certbot'}: subprocess.run(['tools/snap/generate_dnsplugins_all.sh'], check=True, cwd=CERTBOT_DIR) @@ -169,25 +176,29 @@ def main(): with Manager() as manager, Pool(processes=len(targets)) as pool: status = manager.dict() + running = manager.dict({target: True for target in targets}) lock = manager.Lock() - stop_event = Event() - state_process = Process(target=_dump_status, args=(archs, status, stop_event)) - state_process.start() + async_results = [pool.apply_async(_build_snap, (target, archs, status, running, lock)) + for target in targets] - async_results = [pool.apply_async(_build_snap, (target, archs, status, lock)) for target in targets] + process = Process(target=_dump_status, args=(archs, status, running)) + process.start() - workspaces = {} - for async_result in async_results: - workspaces.update(async_result.get()) + try: + process.join(args.timeout) - stop_event.set() - state_process.join() + if process.is_alive(): + raise ValueError(f"Timeout out reached ({args.timeout} seconds) during the build!") - failures = _dump_results(targets, archs, status, workspaces) - _dump_status_final(archs, status) + workspaces = {} + for async_result in async_results: + workspaces.update(async_result.get()) - return 1 if failures else 0 + if _dump_results(targets, archs, status, workspaces): + raise ValueError("There were failures during the build!") + finally: + process.terminate() if __name__ == '__main__': diff --git a/tools/snap/generate_dnsplugins_all.sh b/tools/snap/generate_dnsplugins_all.sh index 6c41a19cd..40404bf9b 100755 --- a/tools/snap/generate_dnsplugins_all.sh +++ b/tools/snap/generate_dnsplugins_all.sh @@ -11,5 +11,6 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do # Create constraints file "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \ <("${CERTBOT_DIR}"/tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt) \ + <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \ > "${PLUGIN_PATH}"/snap-constraints.txt done diff --git a/tools/snap/generate_dnsplugins_snapcraft.sh b/tools/snap/generate_dnsplugins_snapcraft.sh index 06807ec48..d93d8ec73 100755 --- a/tools/snap/generate_dnsplugins_snapcraft.sh +++ b/tools/snap/generate_dnsplugins_snapcraft.sh @@ -23,11 +23,16 @@ parts: ${PLUGIN}: plugin: python source: . - constraints: [\$SNAPCRAFT_PART_SRC/snap-constraints.txt] override-pull: | snapcraftctl pull snapcraftctl set-version \`grep ^version \$SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"\` build-environment: + # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the + # parts.[part_name].constraints option available in snapcraft.yaml when the Python plugin is + # used. This is done to let these constraints be applied not only on the certbot package + # build, but also on any isolated build that pip could trigger when building wheels for + # dependencies. See https://github.com/certbot/certbot/pull/8443 for more info. + - PIP_CONSTRAINT: \$SNAPCRAFT_PART_SRC/snap-constraints.txt - SNAP_BUILD: "True" # To build cryptography and cffi if needed build-packages: [gcc, libffi-dev, libssl-dev, python3-dev] diff --git a/tox.ini b/tox.ini index 7f806bb4d..212d4ee76 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,7 @@ install_packages = source_paths = acme/acme certbot/certbot + certbot-ci/certbot_integration_tests certbot-apache/certbot_apache certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare @@ -188,29 +189,6 @@ whitelist_externals = passenv = DOCKER_* -[testenv:le_auto_centos6] -# At the moment, this tests under Python 2.6 only, as only that version is -# readily available on the CentOS 6 Docker image. -commands = - python {toxinidir}/tests/modification-check.py - docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=centos -t lea letsencrypt-auto-source - docker run --rm -t lea -whitelist_externals = - docker -passenv = - DOCKER_* - TARGET_BRANCH - -[testenv:le_auto_oraclelinux6] -# At the moment, this tests under Python 2.6 only, as only that version is -# readily available on the Oracle Linux 6 Docker image. -commands = - docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=oraclelinux -t lea letsencrypt-auto-source - docker run --rm -t lea -whitelist_externals = - docker -passenv = DOCKER_* - [testenv:docker_dev] # tests the Dockerfile-dev file to ensure development with it works # as expected @@ -245,7 +223,11 @@ commands = {[base]pip_install} acme certbot certbot-dns-rfc2136 certbot-ci pytest certbot-ci/certbot_integration_tests/rfc2136_tests \ --acme-server=pebble --dns-server=bind \ - --numprocesses=1 + --numprocesses=1 \ + --cov=acme --cov=certbot --cov=certbot_dns_rfc2136 --cov-report= \ + --cov-config=certbot-ci/certbot_integration_tests/.coveragerc + coverage report --include 'certbot/*' --show-missing --fail-under=45 + coverage report --include 'certbot-dns-rfc2136/*' --show-missing --fail-under=87 [testenv:integration-external] # Run integration tests with Certbot outside of tox's virtual environment. diff --git a/windows-installer/construct.py b/windows-installer/construct.py index 14f770959..1ce4811ac 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -46,9 +46,11 @@ def _compile_wheels(repo_path, build_path, venv_python): wheels_project = [os.path.join(repo_path, package) for package in certbot_packages] with _prepare_constraints(repo_path) as constraints_file_path: - command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path, '--constraint', constraints_file_path] + env = os.environ.copy() + env['PIP_CONSTRAINT'] = constraints_file_path + command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path] command.extend(wheels_project) - subprocess.check_call(command) + subprocess.check_call(command, env=env) def _prepare_build_tools(venv_path, venv_python, repo_path): @@ -61,15 +63,20 @@ def _prepare_build_tools(venv_path, venv_python, repo_path): @contextlib.contextmanager def _prepare_constraints(repo_path): - requirements = os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'dependency-requirements.txt') - constraints = subprocess.check_output( - [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), requirements], + reqs_certbot = os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'dependency-requirements.txt') + reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt') + constraints_certbot = subprocess.check_output( + [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot], + universal_newlines=True) + constraints_pipstrap = subprocess.check_output( + [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_pipstrap], universal_newlines=True) workdir = tempfile.mkdtemp() try: constraints_file_path = os.path.join(workdir, 'constraints.txt') with open(constraints_file_path, 'a') as file_h: - file_h.write(constraints) + file_h.write(constraints_pipstrap) + file_h.write(constraints_certbot) file_h.write('pywin32=={0}'.format(PYWIN32_VERSION)) yield constraints_file_path finally: