From a00e34345993f0724a05cc2dd0f6e8ea6a4299a5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 15 Jan 2025 09:47:40 -0800 Subject: [PATCH] update policy on minimum dependency versions (#10130) fixes #10105 this PR updates our minimally required cryptography and pyopenssl versions as well as updating our policy for choosing minimum dependency versions generally before this PR, we were trying to keep compatibility with packages available in EPEL 9 using the python 3 version available in RHEL 9. after the discussion in #10105 we decided not to do this anymore because: * EPEL 9 may not want to update to certbot 3.0+ anyway because of our backwards incompatible changes from certbot 2.x * RHEL 9 appstream repos now contain newer versions of many of our dependencies for newer versions of python * alternate installation methods for RHEL 9 based users including our snaps and pip are available on a call we then discussed what distro repositories we should track instead of EPEL 9. our docs previously said Debian sid/unstable, but we felt this as unnecessary because Debian sid can and does change very quickly. if we wanted a new dependency there, Debian could probably accommodate it we also considered RHEL 10 + EPEL 10, however, these repos are not even stable yet and certbot and many of its dependencies are not yet packaged there at all for these reasons, plus many of the reasons we decided to upgrade past EPEL 9 with the default python 3 version there, we decided that at least for now, we will remove any linux distro considerations when choosing minimal dependency versions of certbot as i wrote in the contributing docs, we may choose to reconsider this plan if there are requests for us to do so, but based on the information above, we are not sure this will ever even happen and removing this constraint significantly simplifies development of certbot --- acme/setup.py | 7 ++++--- certbot-nginx/setup.py | 5 +++-- certbot/CHANGELOG.md | 3 ++- certbot/certbot/crypto_util.py | 23 +++------------------ certbot/docs/contributing.rst | 32 +++++++++++------------------ certbot/setup.py | 2 +- tools/oldest_constraints.txt | 8 ++++---- tools/pinning/oldest/pyproject.toml | 12 +++++++++-- tools/requirements.txt | 28 ++++++++++++------------- 9 files changed, 53 insertions(+), 67 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 9bf7e4e28..72a6792a2 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -6,12 +6,13 @@ from setuptools import setup version = '3.2.0.dev0' install_requires = [ - 'cryptography>=3.2.1', + 'cryptography>=42.0.0', # Josepy 2+ may introduce backward incompatible changes by droping usage of # deprecated PyOpenSSL APIs. 'josepy>=1.13.0, <2', - # pyOpenSSL 23.1.0 is a bad release: https://github.com/pyca/pyopenssl/issues/1199 - 'PyOpenSSL>=17.5.0,!=23.1.0', + # PyOpenSSL>=25.0.0 is just needed to satisfy mypy right now so this dependency can probably be + # relaxed to >=24.0.0 if needed. + 'PyOpenSSL>=25.0.0', 'pyrfc3339', 'pytz>=2019.3', 'requests>=2.20.0', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 1abc31408..8d20e5b89 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -9,8 +9,9 @@ install_requires = [ # https://github.com/certbot/certbot/issues/8761 for more info. f'acme>={version}', f'certbot>={version}', - # pyOpenSSL 23.1.0 is a bad release: https://github.com/pyca/pyopenssl/issues/1199 - 'PyOpenSSL>=17.5.0,!=23.1.0', + # PyOpenSSL>=25.0.0 is just needed to satisfy mypy right now so this dependency can probably be + # relaxed to >=24.0.0 if needed. + 'PyOpenSSL>=25.0.0', 'pyparsing>=2.4.7', ] diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index f1da9fe65..434ecc1ec 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -11,7 +11,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * certbot-nginx now requires pyparsing>=2.4.7. -* +* certbot and its acme library now require cryptography>=42.0.0. +* certbot-nginx and our acme library now require pyOpenSSL>=25.0.0. ### Fixed diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 62913bf74..fd77d5342 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -8,7 +8,6 @@ import datetime import hashlib import logging import re -import warnings from typing import List from typing import Optional from typing import Set @@ -171,13 +170,7 @@ def csr_matches_pubkey(csr: bytes, privkey: bytes) -> bool: """ req = x509.load_pem_x509_csr(csr) pkey = serialization.load_pem_private_key(privkey, password=None) - # This would be better written as `req.public_key() == pkey.public_key()`, - # but that requires a newer minimum version of cryptography. - return req.is_signature_valid and req.public_key().public_bytes( - serialization.Encoding.DER, serialization.PublicFormat.SubjectPublicKeyInfo - ) == pkey.public_key().public_bytes( - serialization.Encoding.DER, serialization.PublicFormat.SubjectPublicKeyInfo - ) + return req.is_signature_valid and req.public_key() == pkey.public_key() def import_csr_file( @@ -514,12 +507,7 @@ def notBefore(cert_path: str) -> datetime.datetime: """ with open(cert_path, "rb") as f: cert = x509.load_pem_x509_certificate(f.read()) - # TODO: This should be `not_valid_before_utc` once we raise the minimum - # cryptography version. - # https://github.com/certbot/certbot/issues/10105 - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', message='Properties that return.*datetime object') - return cert.not_valid_before.replace(tzinfo=datetime.timezone.utc) + return cert.not_valid_before_utc def notAfter(cert_path: str) -> datetime.datetime: @@ -533,12 +521,7 @@ def notAfter(cert_path: str) -> datetime.datetime: """ with open(cert_path, "rb") as f: cert = x509.load_pem_x509_certificate(f.read()) - # TODO: This should be `not_valid_after_utc` once we raise the minimum - # cryptography version. - # https://github.com/certbot/certbot/issues/10105 - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', message='Properties that return.*datetime object') - return cert.not_valid_after.replace(tzinfo=datetime.timezone.utc) + return cert.not_valid_after_utc def sha256sum(filename: str) -> str: diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 52cbab72e..18b47f565 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -632,27 +632,19 @@ If you want to learn more about the design used here, see Choosing dependency versions ---------------------------- -A number of Unix distributions create third-party Certbot packages for their users. -Where feasible, the Certbot project tries to manage its dependencies in a way that -does not create avoidable work for packagers. +When choosing dependency versions, we should choose whatever minimum versions +simplify development of Certbot and our own distribution methods such as snaps, +pip, and docker. Since these approaches have full access to PyPI, it's OK if +the required packages declared in ``setup.py`` are quite new. -Avoiding adding new dependencies is a good way to help with this. - -When adding new or upgrading existing Python dependencies, Certbot developers should -pay attention to which distributions are actively packaging Certbot. In particular: - -- EPEL (used by RHEL/CentOS/Fedora) updates Certbot regularly. At the time of writing, - EPEL9 is the release of EPEL where Certbot is being updated, but check the `EPEL - home page `_ and `pkgs.org - `_ for the latest release. -- Debian and Ubuntu only package Certbot when making new releases of their distros. - Checking the available version of dependencies in Debian "sid" and "unstable" can help - to identify dependencies that are likely to be available in the next stable release of - these distros. - -If a dependency is already packaged in these distros and is acceptable for use in Certbot, -the oldest packaged version of that dependency should be chosen and set as the minimum -version in ``setup.py``. +If this approach to development creates significant trouble for some of our users, we +can revisit this decision and weigh their trouble against the difficulties +involved in maintaining support for a wider range of package versions. When +doing this, we should also be sure to consider the feasibility of users getting +access to these newer packages on their system rather than changing our own +approach here. Their OS distribution may be able to package it, especially in +an alternate repository and/or for a different version of Python to help avoid +conflicts with other packages on their system. macOS suggestions ================= diff --git a/certbot/setup.py b/certbot/setup.py index f8c478a01..b18bf27a7 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -30,7 +30,7 @@ install_requires = [ f'acme>={version}', 'ConfigArgParse>=1.5.3', 'configobj>=5.0.6', - 'cryptography>=3.2.1', + 'cryptography>=42.0.0', 'distro>=1.0.1', 'importlib_metadata>=4.6; python_version < "3.10"', # Josepy 2+ may introduce backward incompatible changes by droping usage of diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 9d9d0d6de..bafa13479 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -15,7 +15,7 @@ colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.10" and sys_pl configargparse==1.5.3 ; python_version >= "3.9" and python_version < "3.10" configobj==5.0.6 ; python_version >= "3.9" and python_version < "3.10" coverage==7.6.10 ; python_version >= "3.9" and python_version < "3.10" -cryptography==3.2.1 ; python_version >= "3.9" and python_version < "3.10" +cryptography==42.0.0 ; python_version >= "3.9" and python_version < "3.10" cython==0.29.37 ; python_version >= "3.9" and python_version < "3.10" dill==0.3.9 ; python_version >= "3.9" and python_version < "3.10" distlib==0.3.9 ; python_version >= "3.9" and python_version < "3.10" @@ -55,7 +55,7 @@ pyasn1-modules==0.4.1 ; python_version >= "3.9" and python_version < "3.10" pyasn1==0.4.8 ; python_version >= "3.9" and python_version < "3.10" pycparser==2.14 ; python_version >= "3.9" and python_version < "3.10" pylint==3.3.3 ; python_version >= "3.9" and python_version < "3.10" -pyopenssl==17.5.0 ; python_version >= "3.9" and python_version < "3.10" +pyopenssl==25.0.0 ; python_version >= "3.9" and python_version < "3.10" pyotp==2.9.0 ; python_version >= "3.9" and python_version < "3.10" pyparsing==2.4.7 ; python_version >= "3.9" and python_version < "3.10" pyrfc3339==1.0 ; python_version >= "3.9" and python_version < "3.10" @@ -79,9 +79,9 @@ tldextract==5.1.3 ; python_version >= "3.9" and python_version < "3.10" tomli==2.2.1 ; python_version >= "3.9" and python_version < "3.10" tomlkit==0.13.2 ; python_version >= "3.9" and python_version < "3.10" tox==1.9.2 ; python_version >= "3.9" and python_version < "3.10" -types-cryptography==3.3.23.2 ; python_version >= "3.9" and python_version < "3.10" +types-cffi==1.16.0.20241221 ; python_version >= "3.9" and python_version < "3.10" types-httplib2==0.22.0.20241221 ; python_version >= "3.9" and python_version < "3.10" -types-pyopenssl==23.0.0.0 ; python_version >= "3.9" and python_version < "3.10" +types-pyopenssl==24.1.0.20240722 ; python_version >= "3.9" and python_version < "3.10" types-pyrfc3339==2.0.1.20241107 ; python_version >= "3.9" and python_version < "3.10" types-python-dateutil==2.9.0.20241206 ; python_version >= "3.9" and python_version < "3.10" types-pytz==2024.2.0.20241221 ; python_version >= "3.9" and python_version < "3.10" diff --git a/tools/pinning/oldest/pyproject.toml b/tools/pinning/oldest/pyproject.toml index 23486cc2c..a56befb2f 100644 --- a/tools/pinning/oldest/pyproject.toml +++ b/tools/pinning/oldest/pyproject.toml @@ -1,5 +1,13 @@ +# The purpose of this file is to help us test Certbot against the oldest +# versions of our dependencies which we claim to support in our setup.py files. +# # Security alerts about vulnerable packages in this file can be ignored since # they are only used during testing. +# +# Ideally, generating package pinnings based on our minimum allowed dependency +# versions would be done automatically by tooling, but as of writing this, both +# https://github.com/pypa/pip/issues/8085 and +# https://github.com/python-poetry/poetry/issues/3527 remain unresolved. [tool.poetry] name = "certbot-pinner" version = "0.1.0" @@ -52,7 +60,7 @@ cffi = "1.12.3" chardet = "3.0.4" cloudflare = "1.5.1" configobj = "5.0.6" -cryptography = "3.2.1" +cryptography = "42.0.0" distro = "1.0.1" dns-lexicon = "3.15.1" dnspython = "2.6.1" @@ -67,7 +75,7 @@ ndg-httpsclient = "0.3.2" parsedatetime = "2.4" pbr = "1.8.0" ply = "3.4" -pyOpenSSL = "17.5.0" +pyOpenSSL = "25.0.0" pyRFC3339 = "1.0" pyasn1 = "0.4.8" pycparser = "2.14" diff --git a/tools/requirements.txt b/tools/requirements.txt index 739cbf958..ac8ad11a2 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -16,10 +16,10 @@ babel==2.16.0 ; python_version >= "3.9" and python_version < "4.0" backports-tarfile==1.2.0 ; python_version >= "3.9" and python_version < "3.12" bcrypt==4.2.1 ; python_version >= "3.9" and python_version < "4.0" beautifulsoup4==4.12.3 ; python_version >= "3.9" and python_version < "4.0" -boto3==1.35.93 ; python_version >= "3.9" and python_version < "4.0" -botocore==1.35.93 ; python_version >= "3.9" and python_version < "4.0" +boto3==1.35.99 ; python_version >= "3.9" and python_version < "4.0" +botocore==1.35.99 ; python_version >= "3.9" and python_version < "4.0" build==1.2.2.post1 ; python_version >= "3.9" and python_version < "4.0" -cachecontrol==0.14.1 ; python_version >= "3.9" and python_version < "4.0" +cachecontrol==0.14.2 ; python_version >= "3.9" and python_version < "4.0" cachetools==5.5.0 ; python_version >= "3.9" and python_version < "4.0" certifi==2024.12.14 ; python_version >= "3.9" and python_version < "4.0" cffi==1.17.1 ; python_version >= "3.9" and python_version < "4.0" @@ -50,7 +50,7 @@ fabric==3.2.2 ; python_version >= "3.9" and python_version < "4.0" fastjsonschema==2.21.1 ; python_version >= "3.9" and python_version < "4.0" filelock==3.16.1 ; python_version >= "3.9" and python_version < "4.0" google-api-core==2.24.0 ; python_version >= "3.9" and python_version < "4.0" -google-api-python-client==2.157.0 ; python_version >= "3.9" and python_version < "4.0" +google-api-python-client==2.159.0 ; python_version >= "3.9" and python_version < "4.0" google-auth-httplib2==0.2.0 ; python_version >= "3.9" and python_version < "4.0" google-auth==2.37.0 ; python_version >= "3.9" and python_version < "4.0" googleapis-common-protos==1.66.0 ; python_version >= "3.9" and python_version < "4.0" @@ -81,7 +81,7 @@ markupsafe==3.0.2 ; python_version >= "3.9" and python_version < "4.0" matplotlib-inline==0.1.7 ; python_version >= "3.9" and python_version < "4.0" mccabe==0.7.0 ; python_version >= "3.9" and python_version < "4.0" mdurl==0.1.2 ; python_version >= "3.9" and python_version < "4.0" -more-itertools==10.5.0 ; python_version >= "3.9" and python_version < "4.0" +more-itertools==10.6.0 ; python_version >= "3.9" and python_version < "4.0" msgpack==1.1.0 ; python_version >= "3.9" and python_version < "4.0" msrest==0.7.1 ; python_version >= "3.9" and python_version < "4.0" mypy-extensions==1.0.0 ; python_version >= "3.9" and python_version < "4.0" @@ -98,12 +98,12 @@ pkginfo==1.12.0 ; python_version >= "3.9" and python_version < "4.0" platformdirs==4.3.6 ; python_version >= "3.9" and python_version < "4.0" pluggy==1.5.0 ; python_version >= "3.9" and python_version < "4.0" ply==3.11 ; python_version >= "3.9" and python_version < "4.0" -poetry-core==2.0.0 ; python_version >= "3.9" and python_version < "4.0" -poetry-plugin-export==1.8.0 ; python_version >= "3.9" and python_version < "4.0" -poetry==2.0.0 ; python_version >= "3.9" and python_version < "4.0" +poetry-core==2.0.1 ; python_version >= "3.9" and python_version < "4.0" +poetry-plugin-export==1.9.0 ; python_version >= "3.9" and python_version < "4.0" +poetry==2.0.1 ; python_version >= "3.9" and python_version < "4.0" prompt-toolkit==3.0.48 ; python_version >= "3.9" and python_version < "4.0" proto-plus==1.25.0 ; python_version >= "3.9" and python_version < "4.0" -protobuf==5.29.2 ; python_version >= "3.9" and python_version < "4.0" +protobuf==5.29.3 ; python_version >= "3.9" and python_version < "4.0" ptyprocess==0.7.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform != "win32" pure-eval==0.2.3 ; python_version >= "3.9" and python_version < "4.0" pyasn1-modules==0.4.1 ; python_version >= "3.9" and python_version < "4.0" @@ -112,7 +112,7 @@ pycparser==2.22 ; python_version >= "3.9" and python_version < "4.0" pygments==2.19.1 ; python_version >= "3.9" and python_version < "4.0" pylint==3.3.3 ; python_version >= "3.9" and python_version < "4.0" pynacl==1.5.0 ; python_version >= "3.9" and python_version < "4.0" -pyopenssl==24.3.0 ; python_version >= "3.9" and python_version < "4.0" +pyopenssl==25.0.0 ; python_version >= "3.9" and python_version < "4.0" pyotp==2.9.0 ; python_version >= "3.9" and python_version < "4.0" pyparsing==3.2.1 ; python_version >= "3.9" and python_version < "4.0" pyproject-api==1.8.0 ; python_version >= "3.9" and python_version < "4.0" @@ -141,7 +141,7 @@ s3transfer==0.10.4 ; python_version >= "3.9" and python_version < "4.0" secretstorage==3.3.3 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "linux" semantic-version==2.10.0 ; python_version >= "3.9" and python_version < "4.0" setuptools-rust==1.10.2 ; python_version >= "3.9" and python_version < "4.0" -setuptools==75.7.0 ; python_version >= "3.9" and python_version < "4.0" +setuptools==75.8.0 ; python_version >= "3.9" and python_version < "4.0" shellingham==1.5.4 ; python_version >= "3.9" and python_version < "4.0" six==1.17.0 ; python_version >= "3.9" and python_version < "4.0" snowballstemmer==2.2.0 ; python_version >= "3.9" and python_version < "4.0" @@ -161,7 +161,7 @@ tomli==2.2.1 ; python_version >= "3.9" and python_full_version <= "3.11.0a6" tomlkit==0.13.2 ; python_version >= "3.9" and python_version < "4.0" tox==4.23.2 ; python_version >= "3.9" and python_version < "4.0" traitlets==5.14.3 ; python_version >= "3.9" and python_version < "4.0" -trove-classifiers==2025.1.6.15 ; python_version >= "3.9" and python_version < "4.0" +trove-classifiers==2025.1.10.15 ; python_version >= "3.9" and python_version < "4.0" twine==6.0.1 ; python_version >= "3.9" and python_version < "4.0" types-cffi==1.16.0.20241221 ; python_version >= "3.9" and python_version < "4.0" types-httplib2==0.22.0.20241221 ; python_version >= "3.9" and python_version < "4.0" @@ -171,7 +171,7 @@ types-python-dateutil==2.9.0.20241206 ; python_version >= "3.9" and python_versi types-pytz==2024.2.0.20241221 ; python_version >= "3.9" and python_version < "4.0" types-pywin32==308.0.0.20241221 ; python_version >= "3.9" and python_version < "4.0" types-requests==2.31.0.6 ; python_version >= "3.9" and python_version < "4.0" -types-setuptools==75.6.0.20241223 ; python_version >= "3.9" and python_version < "4.0" +types-setuptools==75.8.0.20250110 ; python_version >= "3.9" and python_version < "4.0" types-urllib3==1.26.25.14 ; python_version >= "3.9" and python_version < "4.0" typing-extensions==4.12.2 ; python_version >= "3.9" and python_version < "4.0" uritemplate==4.1.1 ; python_version >= "3.9" and python_version < "4.0" @@ -179,6 +179,6 @@ urllib3==1.26.20 ; python_version >= "3.9" and python_version < "4.0" virtualenv==20.28.1 ; python_version >= "3.9" and python_version < "4.0" wcwidth==0.2.13 ; python_version >= "3.9" and python_version < "4.0" wheel==0.45.1 ; python_version >= "3.9" and python_version < "4.0" -wrapt==1.17.0 ; python_version >= "3.9" and python_version < "4.0" +wrapt==1.17.2 ; python_version >= "3.9" and python_version < "4.0" xattr==1.1.4 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "darwin" zipp==3.21.0 ; python_version >= "3.9" and python_version < "3.12"