diff --git a/AUTHORS.md b/AUTHORS.md index 88060f42a..f3dca7c69 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -209,6 +209,7 @@ Authors * [Patrick Heppler](https://github.com/PatrickHeppler) * [Paul Buonopane](https://github.com/Zenexer) * [Paul Feitzinger](https://github.com/pfeyz) +* [Paulo Dias](https://github.com/paulojmdias) * [Pavan Gupta](https://github.com/pavgup) * [Pavel Pavlov](https://github.com/ghost355) * [Peter Conrad](https://github.com/pconrad-fb) diff --git a/acme/acme/_internal/tests/test_util.py b/acme/acme/_internal/tests/test_util.py index 9a5de4b25..2ba00d345 100644 --- a/acme/acme/_internal/tests/test_util.py +++ b/acme/acme/_internal/tests/test_util.py @@ -4,20 +4,25 @@ """ import os +import sys from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose from josepy.util import ComparableECKey from OpenSSL import crypto -import pkg_resources + +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources def load_vector(*names): """Load contents of a test vector.""" # luckily, resource_string opens file in binary mode - return pkg_resources.resource_string( - __name__, os.path.join('testdata', *names)) + vector_ref = importlib_resources.files(__package__).joinpath('testdata', *names) + return vector_ref.read_bytes() def _guess_loader(filename, loader_pem, loader_der): diff --git a/acme/setup.py b/acme/setup.py index 8581a1e23..0af48a645 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -22,6 +22,15 @@ docs_extras = [ ] test_extras = [ + # In theory we could scope importlib_resources to env marker 'python_version<"3.9"'. But this + # makes the pinning mechanism emit warnings when running `poetry lock` because in the corner + # case of an extra dependency with env marker coming from a setup.py file, it generate the + # invalid requirement 'importlib_resource>=1.3.1;python<=3.9;extra=="test"'. + # To fix the issue, we do not pass the env marker. This is fine because: + # - importlib_resources can be applied to any Python version, + # - this is a "test" extra dependency for limited audience, + # - it does not change anything at the end for the generated requirement files. + 'importlib_resources>=1.3.1', 'pytest', 'pytest-xdist', 'typing-extensions', diff --git a/certbot-apache/certbot_apache/_internal/apache_util.py b/certbot-apache/certbot_apache/_internal/apache_util.py index 54b9f9824..83bdf4f72 100644 --- a/certbot-apache/certbot_apache/_internal/apache_util.py +++ b/certbot-apache/certbot_apache/_internal/apache_util.py @@ -1,21 +1,28 @@ """ Utility functions for certbot-apache plugin """ +import atexit import binascii import fnmatch import logging import re import subprocess +import sys +from contextlib import ExitStack from typing import Dict from typing import Iterable from typing import List from typing import Optional from typing import Tuple -import pkg_resources - from certbot import errors from certbot import util from certbot.compat import os +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources + + logger = logging.getLogger(__name__) @@ -248,6 +255,8 @@ def find_ssl_apache_conf(prefix: str) -> str: :return: the path the TLS Apache config file :rtype: str """ - return pkg_resources.resource_filename( - "certbot_apache", - os.path.join("_internal", "tls_configs", "{0}-options-ssl-apache.conf".format(prefix))) + file_manager = ExitStack() + atexit.register(file_manager.close) + ref = importlib_resources.files("certbot_apache").joinpath( + "_internal", "tls_configs", "{0}-options-ssl-apache.conf".format(prefix)) + return str(file_manager.enter_context(importlib_resources.as_file(ref))) diff --git a/certbot-apache/certbot_apache/_internal/constants.py b/certbot-apache/certbot_apache/_internal/constants.py index 0861e2da9..8e2aa3769 100644 --- a/certbot-apache/certbot_apache/_internal/constants.py +++ b/certbot-apache/certbot_apache/_internal/constants.py @@ -1,10 +1,14 @@ """Apache plugin constants.""" +import atexit +import sys +from contextlib import ExitStack from typing import Dict from typing import List -import pkg_resources - -from certbot.compat import os +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources MOD_SSL_CONF_DEST = "options-ssl-apache.conf" """Name of the mod_ssl config file as saved @@ -37,8 +41,15 @@ ALL_SSL_OPTIONS_HASHES: List[str] = [ ] """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC""" -AUGEAS_LENS_DIR = pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "augeas_lens")) +def _generate_augeas_lens_dir_static() -> str: + # This code ensures that the resource is accessible as file for the lifetime of current + # Python process, and will be automatically cleaned up on exit. + file_manager = ExitStack() + atexit.register(file_manager.close) + augeas_lens_dir_ref = importlib_resources.files("certbot_apache") / "_internal" / "augeas_lens" + return str(file_manager.enter_context(importlib_resources.as_file(augeas_lens_dir_ref))) + +AUGEAS_LENS_DIR = _generate_augeas_lens_dir_static() """Path to the Augeas lens directory""" REWRITE_HTTPS_ARGS: List[str] = [ diff --git a/certbot-apache/certbot_apache/_internal/tests/configurator_test.py b/certbot-apache/certbot_apache/_internal/tests/configurator_test.py index b625cc198..9f2328e46 100644 --- a/certbot-apache/certbot_apache/_internal/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/_internal/tests/configurator_test.py @@ -5,7 +5,6 @@ import shutil import socket import sys import tempfile -import unittest from unittest import mock import pytest @@ -1659,20 +1658,23 @@ class InstallSslOptionsConfTest(util.ApacheTest): file has been manually edited by the user, and will refuse to update it. This test ensures that all necessary hashes are present. """ - import pkg_resources + if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources + else: # pragma: no cover + import importlib_resources from certbot_apache._internal.constants import ALL_SSL_OPTIONS_HASHES - tls_configs_dir = pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "tls_configs")) - all_files = [os.path.join(tls_configs_dir, name) for name in os.listdir(tls_configs_dir) - if name.endswith('options-ssl-apache.conf')] - assert len(all_files) >= 1 - for one_file in all_files: - file_hash = crypto_util.sha256sum(one_file) - assert file_hash in ALL_SSL_OPTIONS_HASHES, \ - f"Constants.ALL_SSL_OPTIONS_HASHES must be appended with the sha256 " \ - f"hash of {one_file} when it is updated." + ref = importlib_resources.files("certbot_apache") / "_internal" / "tls_configs" + with importlib_resources.as_file(ref) as tls_configs_dir: + all_files = [os.path.join(tls_configs_dir, name) for name in os.listdir(tls_configs_dir) + if name.endswith('options-ssl-apache.conf')] + assert len(all_files) >= 1 + for one_file in all_files: + file_hash = crypto_util.sha256sum(one_file) + assert file_hash in ALL_SSL_OPTIONS_HASHES, \ + f"Constants.ALL_SSL_OPTIONS_HASHES must be appended with the sha256 " \ + f"hash of {one_file} when it is updated." def test_openssl_version(self): self.config._openssl_version = None diff --git a/certbot-apache/certbot_apache/_internal/tests/util.py b/certbot-apache/certbot_apache/_internal/tests/util.py index 57c374f07..5600e8be4 100644 --- a/certbot-apache/certbot_apache/_internal/tests/util.py +++ b/certbot-apache/certbot_apache/_internal/tests/util.py @@ -22,7 +22,7 @@ class ApacheTest(unittest.TestCase): # pylint: disable=arguments-differ self.temp_dir, self.config_dir, self.work_dir = common.dir_setup( test_dir=test_dir, - pkg=__name__) + pkg=__package__) self.config_path = os.path.join(self.temp_dir, config_root) self.vhost_path = os.path.join(self.temp_dir, vhost_root) diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 78ada824c..266693644 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -9,6 +9,7 @@ install_requires = [ # https://github.com/certbot/certbot/issues/8761 for more info. f'acme>={version}', f'certbot>={version}', + 'importlib_resources>=1.3.1; python_version < "3.9"', 'python-augeas', 'setuptools>=41.6.0', ] diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py b/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py index 60f1696a8..583c00465 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py @@ -1,9 +1,15 @@ # -*- coding: utf-8 -*- """General purpose nginx test configuration generator.""" +import atexit import getpass +import sys +from contextlib import ExitStack from typing import Optional -import pkg_resources +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources def construct_nginx_config(nginx_root: str, nginx_webroot: str, http_port: int, https_port: int, @@ -23,10 +29,18 @@ def construct_nginx_config(nginx_root: str, nginx_webroot: str, http_port: int, :return: a string containing the full nginx configuration :rtype: str """ - key_path = key_path if key_path \ - else pkg_resources.resource_filename('certbot_integration_tests', 'assets/key.pem') - cert_path = cert_path if cert_path \ - else pkg_resources.resource_filename('certbot_integration_tests', 'assets/cert.pem') + if not key_path: + file_manager = ExitStack() + atexit.register(file_manager.close) + ref = importlib_resources.files('certbot_integration_tests').joinpath('assets', 'key.pem') + key_path = str(file_manager.enter_context(importlib_resources.as_file(ref))) + + if not cert_path: + file_manager = ExitStack() + atexit.register(file_manager.close) + ref = importlib_resources.files('certbot_integration_tests').joinpath('assets', 'cert.pem') + cert_path = str(file_manager.enter_context(importlib_resources.as_file(ref))) + return '''\ # This error log will be written regardless of server scope error_log # definitions, so we have to set this here in the main scope. diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py index 4a383fe56..d0f60adce 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -1,17 +1,21 @@ """Module to handle the context of RFC2136 integration tests.""" - from contextlib import contextmanager +import sys import tempfile from typing import Generator from typing import Iterable from typing import Tuple -from pkg_resources import resource_filename import pytest from certbot_integration_tests.certbot_tests import context as certbot_context from certbot_integration_tests.utils import certbot_call +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources + class IntegrationTestsContext(certbot_context.IntegrationTestsContext): """Integration test context for certbot-dns-rfc2136""" @@ -44,15 +48,15 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): :yields: Path to credentials file :rtype: str """ - src_file = resource_filename('certbot_integration_tests', - 'assets/bind-config/rfc2136-credentials-{}.ini.tpl' - .format(label)) - - with open(src_file, 'r') as f: - contents = f.read().format( - server_address=self._dns_xdist['address'], - server_port=self._dns_xdist['port'] - ) + src_ref_file = importlib_resources.files('certbot_integration_tests').joinpath( + 'assets', 'bind-config', f'rfc2136-credentials-{label}.ini.tpl' + ) + with importlib_resources.as_file(src_ref_file) as src_file: + with open(src_file, 'r') as f: + contents = f.read().format( + server_address=self._dns_xdist['address'], + server_port=self._dns_xdist['port'] + ) with tempfile.NamedTemporaryFile('w+', prefix='rfc2136-creds-{}'.format(label), suffix='.ini', dir=self.workspace) as fp: diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py index bcec38be2..d05578549 100755 --- a/certbot-ci/certbot_integration_tests/utils/certbot_call.py +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -6,11 +6,8 @@ import subprocess import sys from typing import Dict from typing import List -from typing import Mapping from typing import Tuple -import pkg_resources - import certbot_integration_tests # pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils.constants import * @@ -84,29 +81,14 @@ def _prepare_environ(workspace: str) -> Dict[str, str]: return new_environ -def _compute_additional_args(workspace: str, environ: Mapping[str, str], - force_renew: bool) -> List[str]: - additional_args = [] - output = subprocess.check_output(['certbot', '--version'], - universal_newlines=True, stderr=subprocess.STDOUT, - cwd=workspace, env=environ) - # Typical response is: output = 'certbot 0.31.0.dev0' - version_str = output.split(' ')[1].strip() - if pkg_resources.parse_version(version_str) >= pkg_resources.parse_version('0.30.0'): - additional_args.append('--no-random-sleep-on-renew') - - if force_renew: - additional_args.append('--renew-by-default') - - return additional_args - - def _prepare_args_env(certbot_args: List[str], directory_url: str, http_01_port: int, tls_alpn_01_port: int, config_dir: str, workspace: str, force_renew: bool) -> Tuple[List[str], Dict[str, str]]: new_environ = _prepare_environ(workspace) - additional_args = _compute_additional_args(workspace, new_environ, force_renew) + additional_args = ['--no-random-sleep-on-renew'] + if force_renew: + additional_args.append('--renew-by-default') command = [ 'certbot', diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index 8ec024bb3..2dabf5505 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -15,10 +15,13 @@ from typing import List from typing import Optional from typing import Type -from pkg_resources import resource_filename - from certbot_integration_tests.utils import constants +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources + BIND_DOCKER_IMAGE = "internetsystemsconsortium/bind9:9.16" BIND_BIND_ADDRESS = ("127.0.0.1", 45953) @@ -80,13 +83,12 @@ class DNSServer: def _configure_bind(self) -> None: """Configure the BIND9 server based on the prebaked configuration""" - bind_conf_src = resource_filename( - "certbot_integration_tests", "assets/bind-config" - ) - for directory in ("conf", "zones"): - shutil.copytree( - os.path.join(bind_conf_src, directory), os.path.join(self.bind_root, directory) - ) + ref = importlib_resources.files("certbot_integration_tests") / "assets" / "bind-config" + with importlib_resources.as_file(ref) as path: + for directory in ("conf", "zones"): + shutil.copytree( + os.path.join(path, directory), os.path.join(self.bind_root, directory) + ) def _start_bind(self) -> None: """Launch the BIND9 server as a Docker container""" diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index 9db59d123..8260ccf5e 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -2,6 +2,7 @@ Misc module contains stateless functions that could be used during pytest execution, or outside during setup/teardown of the integration tests environment. """ +import atexit import contextlib import errno import functools @@ -30,13 +31,17 @@ from cryptography.hazmat.primitives.serialization import PrivateFormat from cryptography.x509 import Certificate from cryptography.x509 import load_pem_x509_certificate from OpenSSL import crypto -import pkg_resources import requests from certbot_integration_tests.certbot_tests.context import IntegrationTestsContext from certbot_integration_tests.utils.constants import PEBBLE_ALTERNATE_ROOTS from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources + RSA_KEY_TYPE = 'rsa' ECDSA_KEY_TYPE = 'ecdsa' @@ -125,7 +130,11 @@ def generate_test_file_hooks(config_dir: str, hook_probe: str) -> None: :param str config_dir: current certbot config directory :param str hook_probe: path to the hook probe to test hook scripts execution """ - hook_path = pkg_resources.resource_filename('certbot_integration_tests', 'assets/hook.py') + file_manager = contextlib.ExitStack() + atexit.register(file_manager.close) + hook_path_ref = importlib_resources.files('certbot_integration_tests').joinpath( + 'assets', 'hook.py') + hook_path = str(file_manager.enter_context(importlib_resources.as_file(hook_path_ref))) for hook_dir in list_renewal_hooks_dirs(config_dir): # We want an equivalent of bash `chmod -p $HOOK_DIR, that does not fail if one folder of @@ -260,9 +269,12 @@ def load_sample_data_path(workspace: str) -> str: :returns: the path to the loaded sample data directory :rtype: str """ - original = pkg_resources.resource_filename('certbot_integration_tests', 'assets/sample-config') - copied = os.path.join(workspace, 'sample-config') - shutil.copytree(original, copied, symlinks=True) + original_ref = importlib_resources.files('certbot_integration_tests').joinpath( + 'assets', 'sample-config' + ) + with importlib_resources.as_file(original_ref) as original: + copied = os.path.join(workspace, 'sample-config') + shutil.copytree(original, copied, symlinks=True) if os.name == 'nt': # Fix the symlinks on Windows if GIT is not configured to create them upon checkout diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index 9cd8ddbbe..8214eb761 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -1,33 +1,43 @@ # pylint: disable=missing-module-docstring - +import atexit import json import os import stat +import sys +from contextlib import ExitStack from typing import Tuple -import pkg_resources import requests from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources + PEBBLE_VERSION = 'v2.3.1' -ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') def fetch(workspace: str, http_01_port: int = DEFAULT_HTTP_01_PORT) -> Tuple[str, str, str]: # pylint: disable=missing-function-docstring suffix = 'linux-amd64' if os.name != 'nt' else 'windows-amd64.exe' - pebble_path = _fetch_asset('pebble', suffix) - challtestsrv_path = _fetch_asset('pebble-challtestsrv', suffix) - pebble_config_path = _build_pebble_config(workspace, http_01_port) + file_manager = ExitStack() + atexit.register(file_manager.close) + pebble_path_ref = importlib_resources.files('certbot_integration_tests') / 'assets' + assets_path = str(file_manager.enter_context(importlib_resources.as_file(pebble_path_ref))) + + pebble_path = _fetch_asset('pebble', suffix, assets_path) + challtestsrv_path = _fetch_asset('pebble-challtestsrv', suffix, assets_path) + pebble_config_path = _build_pebble_config(workspace, http_01_port, assets_path) return pebble_path, challtestsrv_path, pebble_config_path -def _fetch_asset(asset: str, suffix: str) -> str: - asset_path = os.path.join(ASSETS_PATH, '{0}_{1}_{2}'.format(asset, PEBBLE_VERSION, suffix)) +def _fetch_asset(asset: str, suffix: str, assets_path: str) -> str: + asset_path = os.path.join(assets_path, '{0}_{1}_{2}'.format(asset, PEBBLE_VERSION, suffix)) if not os.path.exists(asset_path): asset_url = ('https://github.com/letsencrypt/pebble/releases/download/{0}/{1}_{2}' .format(PEBBLE_VERSION, asset, suffix)) @@ -40,15 +50,15 @@ def _fetch_asset(asset: str, suffix: str) -> str: return asset_path -def _build_pebble_config(workspace: str, http_01_port: int) -> str: +def _build_pebble_config(workspace: str, http_01_port: int, assets_path: str) -> str: config_path = os.path.join(workspace, 'pebble-config.json') with open(config_path, 'w') as file_h: file_h.write(json.dumps({ 'pebble': { 'listenAddress': '0.0.0.0:14000', 'managementListenAddress': '0.0.0.0:15000', - 'certificate': os.path.join(ASSETS_PATH, 'cert.pem'), - 'privateKey': os.path.join(ASSETS_PATH, 'key.pem'), + 'certificate': os.path.join(assets_path, 'cert.pem'), + 'privateKey': os.path.join(assets_path, 'key.pem'), 'httpPort': http_01_port, 'tlsPort': 5001, 'ocspResponderURL': 'http://127.0.0.1:{0}'.format(MOCK_OCSP_SERVER_PORT), diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index 0ea188e57..3fddcf104 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -1,20 +1,12 @@ -from pkg_resources import parse_version -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup version = '0.32.0.dev0' -# setuptools 36.2+ is needed for support for environment markers -min_setuptools_version='36.2' -# This conditional isn't necessary, but it provides better error messages to -# people who try to install this package with older versions of setuptools. -if parse_version(setuptools_version) < parse_version(min_setuptools_version): - raise RuntimeError(f'setuptools {min_setuptools_version}+ is required') - install_requires = [ 'coverage', 'cryptography', + 'importlib_resources>=1.3.1; python_version < "3.9"', 'pyopenssl', 'pytest', 'pytest-cov', diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index c2140db14..5fca3f4e6 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -306,7 +306,7 @@ class _GoogleClient: for zone in zones: zone_id = zone['id'] - if 'privateVisibilityConfig' not in zone: + if zone['visibility'] == "public": logger.debug('Found id of %s for %s using name %s', zone_id, domain, zone_name) return zone_id diff --git a/certbot-dns-google/certbot_dns_google/_internal/tests/dns_google_test.py b/certbot-dns-google/certbot_dns_google/_internal/tests/dns_google_test.py index 445c4110f..65315f23f 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/tests/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/_internal/tests/dns_google_test.py @@ -88,6 +88,7 @@ class GoogleClientTest(unittest.TestCase): record_ttl = 42 zone = "ZONE_ID" change = "an-id" + visibility = "public" def _setUp_client_with_mock(self, zone_request_side_effect, rrs_list_side_effect=None): from certbot_dns_google._internal.dns_google import _GoogleClient @@ -183,7 +184,7 @@ class GoogleClientTest(unittest.TestCase): def test_add_txt_record(self, credential_mock): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) - client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) credential_mock.assert_called_once_with('/not/a/real/path.json', scopes=SCOPES) client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) @@ -211,7 +212,7 @@ class GoogleClientTest(unittest.TestCase): def test_add_txt_record_and_poll(self, credential_mock): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) - client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) changes.create.return_value.execute.return_value = {'status': 'pending', 'id': self.change} changes.get.return_value.execute.return_value = {'status': 'done'} @@ -225,6 +226,26 @@ class GoogleClientTest(unittest.TestCase): managedZone=self.zone, project=PROJECT_ID) + @mock.patch('google.auth.load_credentials_from_file') + @mock.patch('certbot_dns_google._internal.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_and_poll_split_horizon(self, credential_mock): + credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) + + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': '{zone}-private'.format(zone=self.zone), 'dnsName': DOMAIN, 'visibility': 'private'},{'id': '{zone}-public'.format(zone=self.zone), 'dnsName': DOMAIN, 'visibility': self.visibility}]}]) + changes.create.return_value.execute.return_value = {'status': 'pending', 'id': self.change} + changes.get.return_value.execute.return_value = {'status': 'done'} + + client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + + changes.create.assert_called_with(body=mock.ANY, + managedZone='{zone}-public'.format(zone=self.zone), + project=PROJECT_ID) + + changes.get.assert_called_with(changeId=self.change, + managedZone='{zone}-public'.format(zone=self.zone), + project=PROJECT_ID) + @mock.patch('google.auth.load_credentials_from_file') @mock.patch('certbot_dns_google._internal.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) @@ -232,7 +253,7 @@ class GoogleClientTest(unittest.TestCase): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) client, changes = self._setUp_client_with_mock( - [{'managedZones': [{'id': self.zone}]}]) + [{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) # pylint: disable=line-too-long mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" with mock.patch(mock_get_rrs) as mock_rrs: @@ -250,7 +271,7 @@ class GoogleClientTest(unittest.TestCase): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) client, changes = self._setUp_client_with_mock( - [{'managedZones': [{'id': self.zone}]}]) + [{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) # pylint: disable=line-too-long mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" with mock.patch(mock_get_rrs) as mock_rrs: @@ -269,7 +290,7 @@ class GoogleClientTest(unittest.TestCase): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) client, changes = self._setUp_client_with_mock( - [{'managedZones': [{'id': self.zone}]}]) + [{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) client.add_txt_record(DOMAIN, "_acme-challenge.example.org", "example-txt-contents", self.record_ttl) assert changes.create.called is False @@ -303,7 +324,7 @@ class GoogleClientTest(unittest.TestCase): def test_add_txt_record_error_during_add(self, credential_mock): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) - client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) changes.create.side_effect = API_ERROR with pytest.raises(errors.PluginError): @@ -315,7 +336,7 @@ class GoogleClientTest(unittest.TestCase): def test_del_txt_record_multi_rrdatas(self, credential_mock): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) - client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) # pylint: disable=line-too-long mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" with mock.patch(mock_get_rrs) as mock_rrs: @@ -356,7 +377,7 @@ class GoogleClientTest(unittest.TestCase): def test_del_txt_record_single_rrdatas(self, credential_mock): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) - client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) # pylint: disable=line-too-long mock_get_rrs = "certbot_dns_google._internal.dns_google._GoogleClient.get_existing_txt_rrset" with mock.patch(mock_get_rrs) as mock_rrs: @@ -408,7 +429,7 @@ class GoogleClientTest(unittest.TestCase): def test_del_txt_record_error_during_delete(self, credential_mock): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) - client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) changes.create.side_effect = API_ERROR client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) @@ -420,7 +441,7 @@ class GoogleClientTest(unittest.TestCase): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) client, unused_changes = self._setUp_client_with_mock( - [{'managedZones': [{'id': self.zone}]}]) + [{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) # Record name mocked in setUp found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") assert found["rrdatas"] == ["\"example-txt-contents\""] @@ -433,7 +454,7 @@ class GoogleClientTest(unittest.TestCase): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) client, unused_changes = self._setUp_client_with_mock( - [{'managedZones': [{'id': self.zone}]}]) + [{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}]) not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") assert not_found is None @@ -444,7 +465,7 @@ class GoogleClientTest(unittest.TestCase): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) client, unused_changes = self._setUp_client_with_mock( - [{'managedZones': [{'id': self.zone}]}], API_ERROR) + [{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}], API_ERROR) # Record name mocked in setUp found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") assert found is None @@ -456,7 +477,7 @@ class GoogleClientTest(unittest.TestCase): credential_mock.return_value = (mock.MagicMock(), PROJECT_ID) client, unused_changes = self._setUp_client_with_mock( - [{'managedZones': [{'id': self.zone}]}], API_ERROR) + [{'managedZones': [{'id': self.zone, 'visibility': self.visibility}]}], API_ERROR) rrset = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") assert not rrset diff --git a/certbot-dns-google/certbot_dns_google/_internal/tests/testdata/discovery.json b/certbot-dns-google/certbot_dns_google/_internal/tests/testdata/discovery.json index 79a406645..9d1f3c319 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/tests/testdata/discovery.json +++ b/certbot-dns-google/certbot_dns_google/_internal/tests/testdata/discovery.json @@ -389,6 +389,11 @@ "items": { "type": "string" } + }, + "visibility": { + "type": "string", + "description": "The zone's visibility: public zones are exposed to the Internet, while private zones are visible only to Virtual Private Cloud resources.", + "default": "public" } } }, diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index ec2806ce3..ebba025e2 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -1,9 +1,12 @@ # pylint: disable=too-many-lines """Nginx Configuration""" +import atexit +from contextlib import ExitStack import logging import re import socket import subprocess +import sys import tempfile import time from typing import Any @@ -21,7 +24,6 @@ from typing import Type from typing import Union import OpenSSL -import pkg_resources from acme import challenges from acme import crypto_util as acme_crypto_util @@ -39,6 +41,11 @@ from certbot_nginx._internal import nginxparser from certbot_nginx._internal import obj from certbot_nginx._internal import parser +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources + NAME_RANK = 0 START_WILDCARD_RANK = 1 END_WILDCARD_RANK = 2 @@ -163,8 +170,12 @@ class NginxConfigurator(common.Configurator): else: config_filename = "options-ssl-nginx-old.conf" - return pkg_resources.resource_filename( - "certbot_nginx", os.path.join("_internal", "tls_configs", config_filename)) + file_manager = ExitStack() + atexit.register(file_manager.close) + ref = importlib_resources.files("certbot_nginx").joinpath( + "_internal", "tls_configs", config_filename) + + return str(file_manager.enter_context(importlib_resources.as_file(ref))) @property def mod_ssl_conf(self) -> str: diff --git a/certbot-nginx/certbot_nginx/_internal/tests/configurator_test.py b/certbot-nginx/certbot_nginx/_internal/tests/configurator_test.py index 7f02f5b3f..24b21edca 100644 --- a/certbot-nginx/certbot_nginx/_internal/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/_internal/tests/configurator_test.py @@ -1,6 +1,5 @@ """Test for certbot_nginx._internal.configurator.""" import sys -import unittest from unittest import mock import OpenSSL @@ -1075,22 +1074,21 @@ class InstallSslOptionsConfTest(util.NginxTest): file has been manually edited by the user, and will refuse to update it. This test ensures that all necessary hashes are present. """ - import pkg_resources - + if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources + else: # pragma: no cover + import importlib_resources + from certbot_nginx._internal.constants import ALL_SSL_OPTIONS_HASHES - all_files = [ - pkg_resources.resource_filename("certbot_nginx", - os.path.join("_internal", "tls_configs", x)) - for x in ("options-ssl-nginx.conf", - "options-ssl-nginx-old.conf", - "options-ssl-nginx-tls12-only.conf") - ] - assert all_files - for one_file in all_files: - file_hash = crypto_util.sha256sum(one_file) - assert file_hash in ALL_SSL_OPTIONS_HASHES, \ - f"Constants.ALL_SSL_OPTIONS_HASHES must be appended with the sha256 " \ - f"hash of {one_file} when it is updated." + + tls_configs_ref = importlib_resources.files("certbot_nginx").joinpath( + "_internal", "tls_configs") + with importlib_resources.as_file(tls_configs_ref) as tls_configs_dir: + for tls_config_file in os.listdir(tls_configs_dir): + file_hash = crypto_util.sha256sum(os.path.join(tls_configs_dir, tls_config_file)) + assert file_hash in ALL_SSL_OPTIONS_HASHES, \ + f"Constants.ALL_SSL_OPTIONS_HASHES must be appended with the sha256 " \ + f"hash of {tls_config_file} when it is updated." def test_nginx_version_uses_correct_config(self): self.config.version = (1, 5, 8) diff --git a/certbot-nginx/certbot_nginx/_internal/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/_internal/tests/nginxparser_test.py index 51a454380..2873b43bd 100644 --- a/certbot-nginx/certbot_nginx/_internal/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/_internal/tests/nginxparser_test.py @@ -70,50 +70,53 @@ class TestRawNginxParser(unittest.TestCase): ' image/jpeg jpg;}}}'.split('\n') def test_parse_from_file(self): - with open(util.get_data_filename('foo.conf')) as handle: - parsed = util.filter_comments(load(handle)) + with util.get_data_filename('foo.conf') as path: + with open(path) as handle: + parsed = util.filter_comments(load(handle)) assert parsed == \ [['user', 'www-data'], - [['http'], - [[['server'], [ - ['listen', '*:80', 'default_server', 'ssl'], - ['server_name', '*.www.foo.com', '*.www.example.com'], - ['root', '/home/ubuntu/sites/foo/'], - [['location', '/status'], [ - [['types'], [['image/jpeg', 'jpg']]], - ]], - [['location', '~', r'case_sensitive\.php$'], [ - ['index', 'index.php'], - ['root', '/var/root'], - ]], - [['location', '~*', r'case_insensitive\.php$'], []], - [['location', '=', r'exact_match\.php$'], []], - [['location', '^~', r'ignore_regex\.php$'], []] - ]]]]] + [['http'], + [[['server'], [ + ['listen', '*:80', 'default_server', 'ssl'], + ['server_name', '*.www.foo.com', '*.www.example.com'], + ['root', '/home/ubuntu/sites/foo/'], + [['location', '/status'], [ + [['types'], [['image/jpeg', 'jpg']]], + ]], + [['location', '~', r'case_sensitive\.php$'], [ + ['index', 'index.php'], + ['root', '/var/root'], + ]], + [['location', '~*', r'case_insensitive\.php$'], []], + [['location', '=', r'exact_match\.php$'], []], + [['location', '^~', r'ignore_regex\.php$'], []] + ]]]]] def test_parse_from_file2(self): - with open(util.get_data_filename('edge_cases.conf')) as handle: - parsed = util.filter_comments(load(handle)) + with util.get_data_filename('edge_cases.conf') as path: + with open(path) as handle: + parsed = util.filter_comments(load(handle)) assert parsed == \ [[['server'], [['server_name', 'simple']]], - [['server'], - [['server_name', 'with.if'], - [['location', '~', '^/services/.+$'], + [['server'], + [['server_name', 'with.if'], + [['location', '~', '^/services/.+$'], [[['if', '($request_filename', '~*', '\\.(ttf|woff)$)'], - [['add_header', 'Access-Control-Allow-Origin', '"*"']]]]]]], - [['server'], - [['server_name', 'with.complicated.headers'], - [['location', '~*', '\\.(?:gif|jpe?g|png)$'], + [['add_header', 'Access-Control-Allow-Origin', '"*"']]]]]]], + [['server'], + [['server_name', 'with.complicated.headers'], + [['location', '~*', '\\.(?:gif|jpe?g|png)$'], [['add_header', 'Pragma', 'public'], - ['add_header', - 'Cache-Control', '\'public, must-revalidate, proxy-revalidate\'', - '"test,;{}"', 'foo'], - ['blah', '"hello;world"'], - ['try_files', '$uri', '@rewrites']]]]]] + ['add_header', + 'Cache-Control', '\'public, must-revalidate, proxy-revalidate\'', + '"test,;{}"', 'foo'], + ['blah', '"hello;world"'], + ['try_files', '$uri', '@rewrites']]]]]] def test_parse_from_file3(self): - with open(util.get_data_filename('multiline_quotes.conf')) as handle: - parsed = util.filter_comments(load(handle)) + with util.get_data_filename('multiline_quotes.conf') as path: + with open(path) as handle: + parsed = util.filter_comments(load(handle)) assert parsed == \ [[['http'], [[['server'], @@ -129,13 +132,15 @@ class TestRawNginxParser(unittest.TestCase): ' end\'']]]]]]]] def test_abort_on_parse_failure(self): - with open(util.get_data_filename('broken.conf')) as handle: - with pytest.raises(ParseException): - load(handle) + with util.get_data_filename('broken.conf') as path: + with open(path) as handle: + with pytest.raises(ParseException): + load(handle) def test_dump_as_file(self): - with open(util.get_data_filename('nginx.conf')) as handle: - parsed = load(handle) + with util.get_data_filename('nginx.conf') as path: + with open(path) as handle: + parsed = load(handle) parsed[-1][-1].append(UnspacedList([['server'], [['listen', ' ', '443', ' ', 'ssl'], ['server_name', ' ', 'localhost'], @@ -155,8 +160,9 @@ class TestRawNginxParser(unittest.TestCase): assert parsed == parsed_new def test_comments(self): - with open(util.get_data_filename('minimalistic_comments.conf')) as handle: - parsed = load(handle) + with util.get_data_filename('minimalistic_comments.conf') as path: + with open(path) as handle: + parsed = load(handle) with tempfile.TemporaryFile(mode='w+t') as f: dump(parsed, f) diff --git a/certbot-nginx/certbot_nginx/_internal/tests/test_util.py b/certbot-nginx/certbot_nginx/_internal/tests/test_util.py index d38c515b4..4f95fe7b2 100644 --- a/certbot-nginx/certbot_nginx/_internal/tests/test_util.py +++ b/certbot-nginx/certbot_nginx/_internal/tests/test_util.py @@ -2,10 +2,11 @@ import copy import shutil import tempfile +import sys +from contextlib import contextmanager from unittest import mock import josepy as jose -import pkg_resources from certbot import util from certbot.compat import os @@ -14,6 +15,10 @@ from certbot.tests import util as test_util from certbot_nginx._internal import configurator from certbot_nginx._internal import nginxparser +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources class NginxTest(test_util.ConfigTestCase): @@ -24,7 +29,7 @@ class NginxTest(test_util.ConfigTestCase): self.config = None self.temp_dir, self.config_dir, self.work_dir = common.dir_setup( - "etc_nginx", __name__) + "etc_nginx", __package__) self.logs_dir = tempfile.mkdtemp('logs') self.config_path = os.path.join(self.temp_dir, "etc_nginx") @@ -78,11 +83,12 @@ class NginxTest(test_util.ConfigTestCase): return config +@contextmanager def get_data_filename(filename): """Gets the filename of a test data file.""" - return pkg_resources.resource_filename( - __name__, os.path.join( - "testdata", "etc_nginx", filename)) + ref = importlib_resources.files(__package__) / "testdata" / "etc_nginx"/ filename + with importlib_resources.as_file(ref) as path: + yield path def filter_comments(tree): diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6aa48c2fc..93995ca0d 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -9,6 +9,7 @@ install_requires = [ # https://github.com/certbot/certbot/issues/8761 for more info. f'acme>={version}', f'certbot>={version}', + 'importlib_resources>=1.3.1; python_version < "3.9"', # pyOpenSSL 23.1.0 is a bad release: https://github.com/pyca/pyopenssl/issues/1199 'PyOpenSSL>=17.5.0,!=23.1.0', 'pyparsing>=2.2.1', diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 7eae5a0a3..2f136c50d 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -18,6 +18,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed * Do not call deprecated datetime.utcnow() and datetime.utcfromtimestamp() +* Filter zones in `certbot-dns-google` to avoid usage of private DNS zones to create records More details about these changes can be found on our GitHub repo. diff --git a/certbot/certbot/_internal/constants.py b/certbot/certbot/_internal/constants.py index 35ee5d54e..680be40fd 100644 --- a/certbot/certbot/_internal/constants.py +++ b/certbot/certbot/_internal/constants.py @@ -1,14 +1,20 @@ """Certbot constants.""" +import atexit import logging +import sys +from contextlib import ExitStack from typing import Any from typing import Dict -import pkg_resources - from acme import challenges from certbot.compat import misc from certbot.compat import os +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources + SETUPTOOLS_PLUGINS_ENTRY_POINT = "certbot.plugins" """Setuptools entry point group name for plugins.""" @@ -220,8 +226,15 @@ SSL_DHPARAMS_DEST = "ssl-dhparams.pem" """Name of the ssl_dhparams file as saved in `certbot.configuration.NamespaceConfig.config_dir`.""" -SSL_DHPARAMS_SRC = pkg_resources.resource_filename( - "certbot", "ssl-dhparams.pem") +def _generate_ssl_dhparams_src_static() -> str: + # This code ensures that the resource is accessible as file for the lifetime of current + # Python process, and will be automatically cleaned up on exit. + file_manager = ExitStack() + atexit.register(file_manager.close) + ssl_dhparams_src_ref = importlib_resources.files("certbot") / "ssl-dhparams.pem" + return str(file_manager.enter_context(importlib_resources.as_file(ssl_dhparams_src_ref))) + +SSL_DHPARAMS_SRC = _generate_ssl_dhparams_src_static() """Path to the nginx ssl_dhparams file found in the Certbot distribution.""" UPDATED_SSL_DHPARAMS_DIGEST = ".updated-ssl-dhparams-pem-digest.txt" diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index a535ac336..2388bbd95 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -22,7 +22,6 @@ from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKey from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.hazmat.primitives.serialization import load_pem_private_key import parsedatetime -import pkg_resources import pytz import certbot @@ -38,12 +37,13 @@ from certbot._internal.plugins import disco as plugins_disco from certbot.compat import filesystem from certbot.compat import os from certbot.plugins import common as plugins_common +from certbot.util import parse_loose_version logger = logging.getLogger(__name__) ALL_FOUR = ("cert", "privkey", "chain", "fullchain") README = "README" -CURRENT_VERSION = pkg_resources.parse_version(certbot.__version__) +CURRENT_VERSION = parse_loose_version(certbot.__version__) BASE_PRIVKEY_MODE = 0o600 # pylint: disable=too-many-lines @@ -492,7 +492,7 @@ class RenewableCert(interfaces.RenewableCert): conf_version = self.configuration.get("version") if (conf_version is not None and - pkg_resources.parse_version(conf_version) > CURRENT_VERSION): + parse_loose_version(conf_version) > CURRENT_VERSION): logger.info( "Attempting to parse the version %s renewal configuration " "file found at %s with version %s of Certbot. This might not " diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index a6086acad..abc30bbdd 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -5,6 +5,7 @@ import argparse import logging import re import shutil +import sys import tempfile from typing import Any from typing import Callable @@ -16,8 +17,6 @@ from typing import Tuple from typing import Type from typing import TypeVar -import pkg_resources - from acme import challenges from certbot import achallenges from certbot import configuration @@ -32,6 +31,11 @@ from certbot.interfaces import Installer as AbstractInstaller from certbot.interfaces import Plugin as AbstractPlugin from certbot.plugins.storage import PluginStorage +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources + logger = logging.getLogger(__name__) @@ -461,10 +465,9 @@ def dir_setup(test_dir: str, pkg: str) -> Tuple[str, str, str]: # pragma: no co filesystem.chmod(config_dir, constants.CONFIG_DIRS_MODE) filesystem.chmod(work_dir, constants.CONFIG_DIRS_MODE) - test_configs = pkg_resources.resource_filename( - pkg, os.path.join("testdata", test_dir)) - - shutil.copytree( - test_configs, os.path.join(temp_dir, test_dir), symlinks=True) + test_dir_ref = importlib_resources.files(pkg).joinpath("testdata", test_dir) + with importlib_resources.as_file(test_dir_ref) as path: + shutil.copytree( + path, os.path.join(temp_dir, test_dir), symlinks=True) return temp_dir, config_dir, work_dir diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index be8e24b55..bbe2ffe9c 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -1,4 +1,6 @@ """Test utilities.""" +import atexit +from contextlib import ExitStack from importlib import reload as reload_module import io import logging @@ -23,7 +25,6 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey import josepy as jose from OpenSSL import crypto -import pkg_resources from certbot import configuration from certbot import util @@ -36,6 +37,11 @@ from certbot.compat import os from certbot.display import util as display_util from certbot.plugins import common +if sys.version_info >= (3, 9): # pragma: no cover + import importlib.resources as importlib_resources +else: # pragma: no cover + import importlib_resources + class DummyInstaller(common.Installer): """Dummy installer plugin for test purpose.""" @@ -75,15 +81,17 @@ class DummyInstaller(common.Installer): def vector_path(*names: str) -> str: """Path to a test vector.""" - return pkg_resources.resource_filename( - __name__, os.path.join('testdata', *names)) + _file_manager = ExitStack() + atexit.register(_file_manager.close) + vector_ref = importlib_resources.files(__package__).joinpath('testdata', *names) + path = _file_manager.enter_context(importlib_resources.as_file(vector_ref)) + return str(path) def load_vector(*names: str) -> bytes: """Load contents of a test vector.""" - # luckily, resource_string opens file in binary mode - data = pkg_resources.resource_string( - __name__, os.path.join('testdata', *names)) + vector_ref = importlib_resources.files(__package__).joinpath('testdata', *names) + data = vector_ref.read_bytes() # Try at most to convert CRLF to LF when data is text try: return data.decode().replace('\r\n', '\n').encode() diff --git a/certbot/setup.py b/certbot/setup.py index 35924ca07..2aeebe9a3 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -1,19 +1,10 @@ import codecs import os import re -import sys -from pkg_resources import parse_version -from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -min_setuptools_version='41.6.0' -# This conditional isn't necessary, but it provides better error messages to -# people who try to install this package with older versions of setuptools. -if parse_version(setuptools_version) < parse_version(min_setuptools_version): - raise RuntimeError(f'setuptools {min_setuptools_version}+ is required') - def read_file(filename, encoding='utf8'): """Read unicode from given file.""" with codecs.open(filename, encoding=encoding) as fd: @@ -44,6 +35,7 @@ install_requires = [ 'configobj>=5.0.6', 'cryptography>=3.2.1', 'distro>=1.0.1', + 'importlib_resources>=1.3.1; python_version < "3.9"', 'josepy>=1.13.0', 'parsedatetime>=2.4', 'pyrfc3339', @@ -51,7 +43,7 @@ install_requires = [ # This dependency needs to be added using environment markers to avoid its # installation on Linux. 'pywin32>=300 ; sys_platform == "win32"', - f'setuptools>={min_setuptools_version}', + 'setuptools>=41.6.0', ] dev_extras = [ diff --git a/pytest.ini b/pytest.ini index 70985484e..46beca405 100644 --- a/pytest.ini +++ b/pytest.ini @@ -31,8 +31,8 @@ # 8) Ignore DeprecationWarning for datetime.utcfromtimestamp() triggered # when importing the pytz.tzinfo module # https://github.com/stub42/pytz/issues/105 -# 9) Ignore DeprecationWarning from boto3 that will drop support for Python 3.7 soon, -# as we will do it also in the same timeline (by December 13, 2023) +# 9) Boto3 is dropping support for Python 3.7 by end of 2023. Let's ignore the associated +# deprecation warning since we will also drop Python 3.7 soon. filterwarnings = error ignore:decodestring\(\) is a deprecated alias:DeprecationWarning:dns @@ -40,7 +40,7 @@ filterwarnings = ignore:'urllib3.contrib.pyopenssl:DeprecationWarning:requests_toolbelt ignore:update_symlinks is deprecated:PendingDeprecationWarning ignore:.*declare_namespace\(':DeprecationWarning - ignore:pkg_resources is deprecated as an API:DeprecationWarning:pkg_resources + ignore:pkg_resources is deprecated as an API:DeprecationWarning ignore:Python 3.7 support will be dropped:PendingDeprecationWarning ignore:datetime.utcfromtimestamp\(\) is deprecated:DeprecationWarning:pytz.tzinfo ignore:Boto3 will no longer support Python 3.7 diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index cec991161..16ba33dbb 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -32,6 +32,7 @@ google-auth==2.16.0 ; python_version >= "3.7" and python_version < "3.8" httplib2==0.9.2 ; python_version >= "3.7" and python_version < "3.8" idna==2.6 ; python_version >= "3.7" and python_version < "3.8" importlib-metadata==6.7.0 ; python_version >= "3.7" and python_version < "3.8" +importlib-resources==1.3.1 ; python_version >= "3.7" and python_version < "3.8" iniconfig==2.0.0 ; python_version >= "3.7" and python_version < "3.8" ipaddress==1.0.16 ; python_version >= "3.7" and python_version < "3.8" isort==5.11.5 ; python_full_version >= "3.7.2" and python_version < "3.8" @@ -61,7 +62,7 @@ pyparsing==2.2.1 ; python_version >= "3.7" and python_version < "3.8" pyrfc3339==1.0 ; python_version >= "3.7" and python_version < "3.8" pytest-cov==4.1.0 ; python_version >= "3.7" and python_version < "3.8" pytest-xdist==3.3.1 ; python_version >= "3.7" and python_version < "3.8" -pytest==7.4.0 ; python_version >= "3.7" and python_version < "3.8" +pytest==7.4.2 ; python_version >= "3.7" and python_version < "3.8" python-augeas==0.5.0 ; python_version >= "3.7" and python_version < "3.8" python-dateutil==2.8.2 ; python_version >= "3.7" and python_version < "3.8" python-digitalocean==1.11 ; python_version >= "3.7" and python_version < "3.8" @@ -75,7 +76,7 @@ s3transfer==0.3.7 ; python_version >= "3.7" and python_version < "3.8" setuptools==41.6.0 ; python_version >= "3.7" and python_version < "3.8" six==1.11.0 ; python_version >= "3.7" and python_version < "3.8" soupsieve==2.4.1 ; python_version >= "3.7" and python_version < "3.8" -tldextract==3.4.4 ; python_version >= "3.7" and python_version < "3.8" +tldextract==3.5.0 ; python_version >= "3.7" and python_version < "3.8" tomli==2.0.1 ; python_version >= "3.7" and python_version < "3.8" tomlkit==0.12.1 ; python_full_version >= "3.7.2" and python_version < "3.8" tox==1.9.2 ; python_version >= "3.7" and python_version < "3.8" @@ -88,13 +89,13 @@ types-python-dateutil==2.8.19.14 ; python_version >= "3.7" and python_version < types-pytz==2023.3.0.1 ; python_version >= "3.7" and python_version < "3.8" types-pywin32==306.0.0.4 ; python_version >= "3.7" and python_version < "3.8" types-requests==2.31.0.2 ; python_version >= "3.7" and python_version < "3.8" -types-setuptools==68.1.0.0 ; python_version >= "3.7" and python_version < "3.8" +types-setuptools==68.2.0.0 ; python_version >= "3.7" and python_version < "3.8" types-six==1.16.21.9 ; python_version >= "3.7" and python_version < "3.8" types-urllib3==1.26.25.14 ; python_version >= "3.7" and python_version < "3.8" typing-extensions==4.7.1 ; python_version >= "3.7" and python_version < "3.8" uritemplate==3.0.1 ; python_version >= "3.7" and python_version < "3.8" urllib3==1.24.2 ; python_version >= "3.7" and python_version < "3.8" -virtualenv==20.24.3 ; python_version >= "3.7" and python_version < "3.8" +virtualenv==20.24.5 ; python_version >= "3.7" and python_version < "3.8" wheel==0.33.6 ; python_version >= "3.7" and python_version < "3.8" wrapt==1.15.0 ; python_full_version >= "3.7.2" and python_version < "3.8" zipp==3.15.0 ; python_version >= "3.7" and python_version < "3.8" diff --git a/tools/pinning/oldest/pyproject.toml b/tools/pinning/oldest/pyproject.toml index 469594f38..37dcd15b1 100644 --- a/tools/pinning/oldest/pyproject.toml +++ b/tools/pinning/oldest/pyproject.toml @@ -61,6 +61,7 @@ google-api-python-client = "1.6.5" google-auth = "2.16.0" httplib2 = "0.9.2" idna = "2.6" +importlib-resources = "1.3.1" ipaddress = "1.0.16" ndg-httpsclient = "0.3.2" parsedatetime = "2.4" diff --git a/tools/requirements.txt b/tools/requirements.txt index 9756977ea..9c9cc333f 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -10,7 +10,7 @@ apacheconfig==0.3.2 ; python_version >= "3.7" and python_version < "4.0" appnope==0.1.3 ; python_version >= "3.7" and python_version < "4.0" and sys_platform == "darwin" astroid==2.13.5 ; python_full_version >= "3.7.2" and python_version < "4.0" attrs==23.1.0 ; python_version >= "3.7" and python_version < "4.0" -azure-core==1.29.3 ; python_version >= "3.7" and python_version < "4.0" +azure-core==1.29.4 ; python_version >= "3.7" and python_version < "4.0" azure-devops==7.1.0b3 ; python_version >= "3.7" and python_version < "4.0" babel==2.12.1 ; python_version >= "3.7" and python_version < "4.0" backcall==0.2.0 ; python_version >= "3.7" and python_version < "4.0" @@ -18,8 +18,8 @@ backports-cached-property==1.0.2 ; python_version >= "3.7" and python_version < bcrypt==4.0.1 ; python_version >= "3.7" and python_version < "4.0" beautifulsoup4==4.12.2 ; python_version >= "3.7" and python_version < "4.0" bleach==6.0.0 ; python_version >= "3.7" and python_version < "4.0" -boto3==1.28.35 ; python_version >= "3.7" and python_version < "4.0" -botocore==1.31.35 ; python_version >= "3.7" and python_version < "4.0" +boto3==1.28.45 ; python_version >= "3.7" and python_version < "4.0" +botocore==1.31.45 ; python_version >= "3.7" and python_version < "4.0" cachecontrol==0.12.14 ; python_version >= "3.7" and python_version < "4.0" cachetools==5.3.1 ; python_version >= "3.7" and python_version < "4.0" cachy==0.3.0 ; python_version >= "3.7" and python_version < "4.0" @@ -42,23 +42,23 @@ distlib==0.3.7 ; python_version >= "3.7" and python_version < "4.0" distro==1.8.0 ; python_version >= "3.7" and python_version < "4.0" dns-lexicon==3.14.1 ; python_version >= "3.7" and python_version < "4.0" dnspython==2.3.0 ; python_version >= "3.7" and python_version < "4.0" -docutils==0.18.1 ; python_version >= "3.7" and python_version < "4.0" +docutils==0.19 ; python_version >= "3.7" and python_version < "4.0" dulwich==0.20.50 ; python_version >= "3.7" and python_version < "4.0" exceptiongroup==1.1.3 ; python_version >= "3.7" and python_version < "3.11" execnet==2.0.2 ; python_version >= "3.7" and python_version < "4.0" -fabric==3.2.1 ; python_version >= "3.7" and python_version < "4.0" +fabric==3.2.2 ; python_version >= "3.7" and python_version < "4.0" filelock==3.12.2 ; python_version >= "3.7" and python_version < "4.0" google-api-core==2.11.1 ; python_version >= "3.7" and python_version < "4.0" -google-api-python-client==2.97.0 ; python_version >= "3.7" and python_version < "4.0" -google-auth-httplib2==0.1.0 ; python_version >= "3.7" and python_version < "4.0" -google-auth==2.22.0 ; python_version >= "3.7" and python_version < "4.0" +google-api-python-client==2.98.0 ; python_version >= "3.7" and python_version < "4.0" +google-auth-httplib2==0.1.1 ; python_version >= "3.7" and python_version < "4.0" +google-auth==2.23.0 ; python_version >= "3.7" and python_version < "4.0" googleapis-common-protos==1.60.0 ; python_version >= "3.7" and python_version < "4.0" html5lib==1.1 ; python_version >= "3.7" and python_version < "4.0" httplib2==0.22.0 ; python_version >= "3.7" and python_version < "4.0" idna==3.4 ; python_version >= "3.7" and python_version < "4.0" imagesize==1.4.1 ; python_version >= "3.7" and python_version < "4.0" importlib-metadata==4.13.0 ; python_version >= "3.7" and python_version < "4.0" -importlib-resources==5.12.0 ; python_version >= "3.7" and python_version < "3.9" +importlib-resources==5.12.0 ; python_version >= "3.7" and python_version < "4.0" iniconfig==2.0.0 ; python_version >= "3.7" and python_version < "4.0" invoke==2.2.0 ; python_version >= "3.7" and python_version < "4.0" ipdb==0.13.13 ; python_version >= "3.7" and python_version < "4.0" @@ -104,7 +104,7 @@ poetry-core==1.3.2 ; python_version >= "3.7" and python_version < "4.0" poetry-plugin-export==1.2.0 ; python_version >= "3.7" and python_version < "4.0" poetry==1.2.2 ; python_version >= "3.7" and python_version < "4.0" prompt-toolkit==3.0.39 ; python_version >= "3.7" and python_version < "4.0" -protobuf==4.24.1 ; python_version >= "3.7" and python_version < "4.0" +protobuf==4.24.3 ; python_version >= "3.7" and python_version < "4.0" ptyprocess==0.7.0 ; python_version >= "3.7" and python_version < "4.0" py==1.11.0 ; python_version >= "3.7" and python_version < "4.0" pyasn1-modules==0.3.0 ; python_version >= "3.7" and python_version < "4.0" @@ -121,11 +121,11 @@ pyrfc3339==1.1 ; python_version >= "3.7" and python_version < "4.0" pyrsistent==0.19.3 ; python_version >= "3.7" and python_version < "4.0" pytest-cov==4.1.0 ; python_version >= "3.7" and python_version < "4.0" pytest-xdist==3.3.1 ; python_version >= "3.7" and python_version < "4.0" -pytest==7.4.0 ; python_version >= "3.7" and python_version < "4.0" +pytest==7.4.2 ; python_version >= "3.7" and python_version < "4.0" python-augeas==1.1.0 ; python_version >= "3.7" and python_version < "4.0" python-dateutil==2.8.2 ; python_version >= "3.7" and python_version < "4.0" python-digitalocean==1.17.0 ; python_version >= "3.7" and python_version < "4.0" -pytz==2023.3 ; python_version >= "3.7" and python_version < "4.0" +pytz==2023.3.post1 ; python_version >= "3.7" and python_version < "4.0" pywin32-ctypes==0.2.2 ; python_version >= "3.7" and python_version < "4.0" and sys_platform == "win32" pywin32==306 ; python_version >= "3.7" and python_version < "4.0" and sys_platform == "win32" pyyaml==6.0.1 ; python_version >= "3.7" and python_version < "4.0" @@ -147,17 +147,16 @@ shellingham==1.5.3 ; python_version >= "3.7" and python_version < "4.0" six==1.16.0 ; python_version >= "3.7" and python_version < "4.0" snowballstemmer==2.2.0 ; python_version >= "3.7" and python_version < "4.0" soupsieve==2.4.1 ; python_version >= "3.7" and python_version < "4.0" -sphinx-rtd-theme==1.3.0 ; python_version >= "3.7" and python_version < "4.0" +sphinx-rtd-theme==0.5.1 ; python_version >= "3.7" and python_version < "4.0" sphinx==5.3.0 ; python_version >= "3.7" and python_version < "4.0" sphinxcontrib-applehelp==1.0.2 ; python_version >= "3.7" and python_version < "4.0" sphinxcontrib-devhelp==1.0.2 ; python_version >= "3.7" and python_version < "4.0" sphinxcontrib-htmlhelp==2.0.0 ; python_version >= "3.7" and python_version < "4.0" -sphinxcontrib-jquery==4.1 ; python_version >= "3.7" and python_version < "4.0" sphinxcontrib-jsmath==1.0.1 ; python_version >= "3.7" and python_version < "4.0" sphinxcontrib-qthelp==1.0.3 ; python_version >= "3.7" and python_version < "4.0" sphinxcontrib-serializinghtml==1.1.5 ; python_version >= "3.7" and python_version < "4.0" -tldextract==3.4.4 ; python_version >= "3.7" and python_version < "4.0" -tomli==2.0.1 ; python_version >= "3.7" and python_version < "3.11" +tldextract==3.5.0 ; python_version >= "3.7" and python_version < "4.0" +tomli==2.0.1 ; python_version >= "3.7" and python_full_version <= "3.11.0a6" tomlkit==0.12.1 ; python_version >= "3.7" and python_version < "4.0" tox==3.28.0 ; python_version >= "3.7" and python_version < "4.0" traitlets==5.9.0 ; python_version >= "3.7" and python_version < "4.0" @@ -170,7 +169,7 @@ types-python-dateutil==2.8.19.14 ; python_version >= "3.7" and python_version < types-pytz==2023.3.0.1 ; python_version >= "3.7" and python_version < "4.0" types-pywin32==306.0.0.4 ; python_version >= "3.7" and python_version < "4.0" types-requests==2.31.0.2 ; python_version >= "3.7" and python_version < "4.0" -types-setuptools==68.1.0.0 ; python_version >= "3.7" and python_version < "4.0" +types-setuptools==68.2.0.0 ; python_version >= "3.7" and python_version < "4.0" types-six==1.16.21.9 ; python_version >= "3.7" and python_version < "4.0" types-urllib3==1.26.25.14 ; python_version >= "3.7" and python_version < "4.0" typing-extensions==4.7.1 ; python_version >= "3.7" and python_version < "4.0"