From 2acc1dcc8997b75589131247661daeac08360b0f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 1 Jun 2020 15:18:38 -0700 Subject: [PATCH] Fix TLS-ALPN tests with newer versions of OpenSSL (#8026) Fixes #7988. As described there, the steps involved are: 1. Update our tests so they fail due to this problem. 2. Update the keys used in the tests so they pass with the new changes. For 1, see a [failing travis run](https://travis-ci.com/github/certbot/certbot/jobs/340710511) with the included change. And for the full output to confirm that this is what is failing, see a [run on debian 10](https://github.com/certbot/certbot/files/4692350/debian_run_log.txt). This PR adds `rsa4096_key.pem` and `rsa4096_cert.pem`, updates the `TLS-ALPN` test to use those keys in place of the 1024-bit versions, and fixes the README in that `testdata` folder with correct instructions to generate these files. * export PIP_NO_BINARY in pip install subshell in test_sdists.sh * set environment variable on the line that installs most packages * Generate 4096-bit rsa key and cert, and fix README instructions to do so. * Update TLS_ALPN test to use 4096-bit key instead of 1024-bit key. * Update changelog * Older versions of Python have an error when both VIRTUAL_NO_DOWNLOAD and PIP_NO_BINARY are set, so only apply the latter at the install phase. * Add enum34 constraint manually, since rebuild_dependencies.py seems to be broken. * only delete key if it exists * Check OpenSSL version before trying to set PIP_NO_BINARY * Add comment explaining why we only set PIP_NO_BINARY at the install step --- acme/tests/standalone_test.py | 4 +- acme/tests/testdata/README | 2 +- acme/tests/testdata/rsa4096_cert.pem | 30 +++++++++++ acme/tests/testdata/rsa4096_key.pem | 51 +++++++++++++++++++ certbot/CHANGELOG.md | 1 + letsencrypt-auto-source/letsencrypt-auto | 4 +- .../pieces/dependency-requirements.txt | 4 +- tests/letstest/scripts/test_sdists.sh | 2 +- tools/_venv_common.py | 30 +++++++++++ 9 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 acme/tests/testdata/rsa4096_cert.pem create mode 100644 acme/tests/testdata/rsa4096_key.pem diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index d03b56535..3d068fb46 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -122,8 +122,8 @@ class TLSALPN01ServerTest(unittest.TestCase): )} # Use different certificate for challenge. self.challenge_certs = {b'localhost': ( - test_util.load_pyopenssl_private_key('rsa1024_key.pem'), - test_util.load_cert('rsa1024_cert.pem'), + test_util.load_pyopenssl_private_key('rsa4096_key.pem'), + test_util.load_cert('rsa4096_cert.pem'), )} from acme.standalone import TLSALPN01Server self.server = TLSALPN01Server(("localhost", 0), certs=self.certs, diff --git a/acme/tests/testdata/README b/acme/tests/testdata/README index d65cc3018..9c1a12d80 100644 --- a/acme/tests/testdata/README +++ b/acme/tests/testdata/README @@ -4,7 +4,7 @@ to use appropriate extension for vector filenames: .pem for PEM and The following command has been used to generate test keys: - for x in 256 512 1024 2048; do openssl genrsa -out rsa${k}_key.pem $k; done + for k in 256 512 1024 2048 4096; do openssl genrsa -out rsa${k}_key.pem $k; done and for the CSR: diff --git a/acme/tests/testdata/rsa4096_cert.pem b/acme/tests/testdata/rsa4096_cert.pem new file mode 100644 index 000000000..d8f6a9896 --- /dev/null +++ b/acme/tests/testdata/rsa4096_cert.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFDTCCAvWgAwIBAgIUImqDrP53V69vFROsjP/gL0YtoA4wDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wHhcNMjAwNTI3MjMyNDE0WhcNMjAw +NjI2MjMyNDE0WjAWMRQwEgYDVQQDDAtleGFtcGxlLmNvbTCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBANY9LKLk9Dxn0MUMQFHwBoTN4ehDSWBws2KcytpF +mc8m9Mfk1wmb4fQSKYtK3wIFMfIyo9HQu0nKqMkkUw52o3ZXyOv+oWwF5qNy2BKu +lh5OMSkaZ0o13zoPpW42e+IUnyxvg70+0urD+sUue4cyTHh/nBIUjrM/05ZJ/ac8 +HR0RK3H41YoqBjq69JjMZczZZhbNFit3s6p0R1TbVAgc3ckqbtX5BDyQMQQCP4Ed +m4DgbAFVqdcPUCC5W3F3fmuQiPKHiADzONZnXpy6lUvLDWqcd6loKp+nKHM6OkXX +8hmD7pE1PYMQo4hqOfhBR2IgMjAShwd5qUFjl1m2oo0Qm3PFXOk6i2ZQdS6AA/yd +B5/mX0RnM2oIdFZPb6UZFSmtEgs9sTzn+hMUyNSZQRE54px1ur1xws2R+vbsCyM5 ++KoFVxDjVjU9TlZx3GvDvnqz/tbHjji6l8VHZYOBMBUXbKHu2U6pJFZ5Zp7k68/z +a3Fb9Pjtn3iRkXEyC0N5kLgqO4QTlExnxebV8aMvQpWd/qefnMn9qPYIZPEXSQAR +mEBIahkcACb60s+acG0WFFluwBPtBqEr8Q67XlSF0Ibf4iBiRzpPobhlWta1nrFg +4IWHMSoZ0PE75bhIGBEkhrpcXQCAxXmAfxfjKDH7jdJ1fRdnZ/9+OzwYGVX5GH/l +0QDtAgMBAAGjUzBRMB0GA1UdDgQWBBQh3xiz/o1nEU2ySylZ9gxCXvIPGzAfBgNV +HSMEGDAWgBQh3xiz/o1nEU2ySylZ9gxCXvIPGzAPBgNVHRMBAf8EBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4ICAQAELoXz31oR9pdAwidlv9ZBOKiC7KBWy8VMqXNVkfTn +bVRxAUex7zleLFIOkWnqadsMesU9sIwrbLzBcZ8Q/vBY+z2xOPdXcgcAoAmdKWoq +YBQNiqng9r54sqlzB/77QZCf5fdktESe7NTxhCifgx5SAWq7IUQs/lm3tnMUSAfE +5ctuN6M+w8K54y3WDprcfMHpnc3ZHeSPhVQApHM0h/bDvXq0bRS7kmq27Hb153Qm +nH3TwYB5pPSWW38NbUc+s/a7mItO7S8ly8yGbA0j9c/IbN5lM+OCdk06asz3+c8E +uo8nuCBoYO5+6AqC2N7WJ3Tdr/pFA8jTbd6VNVlgCWTIR8ZosL5Fgkfv+4fUBrHt +zdVUqMUzvga5rvZnwnJ5Qfu/drHeAAo9MTNFQNe2QgDlYfWBh5GweolgmFSwrpkY +v/5wLtIyv/ASHKswybbqMIlpttcLTXjx5yuh8swttT6Wh+FQqqQ32KSRB3StiwyK +oH0ZhrwYHiFYNlPxecGX6XUta6rFtTlEdkBGSnXzgiTzL2l+Nc0as0V5B9RninZG +qJ+VOChSQ0OFvg1riSXv7tMvbLdGQnxwTRL3t6BMS8I4LA2m3ZfWUcuXT783ODTH +16f1Q1AgXd2csstTWO9cv+N/0fpX31nqrm6+CrGduSr2u4HjYYnlLIUhmdTvK3fX +Fg== +-----END CERTIFICATE----- diff --git a/acme/tests/testdata/rsa4096_key.pem b/acme/tests/testdata/rsa4096_key.pem new file mode 100644 index 000000000..b16e1fc9b --- /dev/null +++ b/acme/tests/testdata/rsa4096_key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKgIBAAKCAgEA1j0souT0PGfQxQxAUfAGhM3h6ENJYHCzYpzK2kWZzyb0x+TX +CZvh9BIpi0rfAgUx8jKj0dC7ScqoySRTDnajdlfI6/6hbAXmo3LYEq6WHk4xKRpn +SjXfOg+lbjZ74hSfLG+DvT7S6sP6xS57hzJMeH+cEhSOsz/Tlkn9pzwdHRErcfjV +iioGOrr0mMxlzNlmFs0WK3ezqnRHVNtUCBzdySpu1fkEPJAxBAI/gR2bgOBsAVWp +1w9QILlbcXd+a5CI8oeIAPM41mdenLqVS8sNapx3qWgqn6coczo6RdfyGYPukTU9 +gxCjiGo5+EFHYiAyMBKHB3mpQWOXWbaijRCbc8Vc6TqLZlB1LoAD/J0Hn+ZfRGcz +agh0Vk9vpRkVKa0SCz2xPOf6ExTI1JlBETninHW6vXHCzZH69uwLIzn4qgVXEONW +NT1OVnHca8O+erP+1seOOLqXxUdlg4EwFRdsoe7ZTqkkVnlmnuTrz/NrcVv0+O2f +eJGRcTILQ3mQuCo7hBOUTGfF5tXxoy9ClZ3+p5+cyf2o9ghk8RdJABGYQEhqGRwA +JvrSz5pwbRYUWW7AE+0GoSvxDrteVIXQht/iIGJHOk+huGVa1rWesWDghYcxKhnQ +8TvluEgYESSGulxdAIDFeYB/F+MoMfuN0nV9F2dn/347PBgZVfkYf+XRAO0CAwEA +AQKCAgEA0hZdTkQtCYtYm9LexDsXeWYX8VcCfrMmBj7xYcg9A3oVMmzDPuYBVwH0 +gWbjd6y2hOaJ5TfGYZ99kvmvBRDsTSHaoyopC7BhssjtAKz6Ay/0X3VH8usPQ3WS +aZi+NT65tK6KRqtz08ppgLGLa1G00bl5x/Um1rpxeACI4FU/y4BJ1VMJvJpnT3KE +Z86Qyagqx5NH+UpCApZSWPFX3zjHePzGgcfXErjniCHYOnpZQrFQ2KIzkfSvQ9fg +x01ByKOM2CB2C1B33TCzBAioXRH6zyAu7A59NeCK9ywTduhDvie1a+oEryFC7IQW +4s7I/H3MGX4hsf/pLXlHMy+5CZJOjRaC2h+pypfbbcuiXu6Sn64kHNpiI7SxI5DI +MIRjyG7MdUcrzq0Rt8ogwwpbCoRqrl/w3bhxtqmeZaEZtyxbjlm7reK2YkIFDgyz +JMqiJK5ZAi+9L/8c0xhjjAQQ0sIzrjmjA8U+6YnWL9jU5qXTVnBB8XQucyeeZGgk +yRHyMur71qOXN8z3UEva7MHkDTUBlj8DgTz6sEjqCipaWl0CXfDNa4IhHIXD5qiF +wplhq7OeS0v6EGG/UFa3Q/lFntxtrayxJX7uvvSccGzjPKXTjpWUELLi/FdnIsum +eXT3RgIEYozj4BibDXaBLfHTCVzxOr7AAEvKM9XWSUgLA0paSWECggEBAO9ZBeE1 +GWzd1ejTTkcxBC9AK2rNsYG8PdNqiof/iTbuJWNeRqpG+KB/0CNIpjZ2X5xZd0tM +FDpHTFehlP26Roxuq50iRAFc+SN5KoiO0A3JuJAidreIgRTia1saUUrypHqWrYEA +VZVj2AI8Pyg3s1OkR2frFskY7hXBVb/pJNDP/m9xTXXIYiIXYkHYe+4RIJCnAxRv +q5YHKaX+0Ull9YCZJCxmwvcHat8sgu8qkiwUMEM6QSNEkrEbdnWYBABvC1AR6sws +7MP1h9+j22n4Zc/3D6kpFZEL9Erx8nNyhbOZ6q2Tdnf6YKVVjZdyVa8VyNnR0ROl +3BjkFaHb/bg4e4kCggEBAOUk8ZJS3qBeGCOjug384zbHGcnhUBYtYJiOz+RXBtP+ +PRksbFtTkgk1sHuSGO8YRddU4Qv7Av1xL8o+DEsLBSD0YQ7pmLrR/LK+iDQ5N63O +Fve9uJH0ybxAOkiua7G24+lTsVUP//KWToL4Wh5zbHBBjL5D2Z9zoeVbcE87xhva +lImMVr4Ex252DqNP9wkZxBjudFyJ/C/TnXrjPcgwhxWTC7sLQMhE5p+490G7c4hX +PywkIKrANbu37KDiAvVS+dC66ZgpL/NUDkeloAmGNO08LGzbV6YKchlvDyWU/AvW +0hYjbL0FUq7K/wp1G9fumolB+fbI25K9c13X93STzUUCggEBAJDsNFUyk5yJjbYW +C/WrRj9d+WwH9Az77+uNPSgvn+O0usq6EMuVgYGdImfa21lqv2Wp/kOHY1AOT7lX +yyD+oyzw7dSNJOQ2aVwDR6+72Vof5DLRy1RBwPbmSd61xrc8yD658YCEtU1pUSe5 +VvyBDYH9nIbdn8RP5gkiMUusXXBaIFNWJXLFzDWcNxBrhk6V7EPp/EFphFmpKJyr ++AkbRVWCZJbF+hMdWKadCwLJogwyhS6PnVU/dhrq6AU38GRa2Fy5HJRYN1xH1Oej +DX3Su8L6c28Xw0k6FcczTHx+wVoIPkKvYTIwVkiFzt/+iMckx6KsGo5tBSHFKRwC +WlQrTxECggEBALjUruLnY1oZ7AC7bTUhOimSOfQEgTQSUCtebsRxijlvhtsKYTDd +XRt+qidStjgN7S/+8DRYuZWzOeg5WnMhpXZqiOudcyume922IGl3ibjxVsdoyjs5 +J4xohlrgDlBgBMDNWGoTqNGFejjcmNydH+gAh8VlN2INxJYbxqCyx17qVgwJHmLR +uggYxD/pHYvCs9GkbknCp5/wYsOgDtKuihfV741lS1D/esN1UEQ+LrfYIEW7snno +5q7Pcdhn1hkKYCWEzy2Ec4Aj2gzixQ9JqOF/OxpnZvCw1k47rg0TeqcWFYnz8x8Y +7xO8/DH0OoxXk2GJzVXJuItJs4gLzzfCjL0CggEAJFHfC9jisdy7CoWiOpNCSF1B +S0/CWDz77cZdlWkpTdaXGGp1MA/UKUFPIH8sOHfvpKS660+X4G/1ZBHmFb4P5kFF +Qy8UyUMKtSOEdZS6KFlRlfSCAMd5aSTmCvq4OSjYEpMRwUhU/iEJNkn9Z1Soehe0 +U3dxJ8KiT1071geO6rRquSHoSJs6Y0WQKriYYQJOhh4Axs3PQihER2eyh+WGk8YJ +02m0mMsjntqnXtdc6IcdKaHp9ko+OpM9QZLsvt19fxBcrXj/i21uUXrzuNtKfO6M +JqGhsOrO2dh8lMhvodENvgKA0DmYDC9N7ogo7bxTNSedcjBF46FhJoqii8m70Q== +-----END RSA PRIVATE KEY----- diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 8e5a7bc78..4b8a68e04 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -16,6 +16,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Add support for OCSP responses which use a public key hash ResponderID, fixing interoperability with Sectigo CAs. +* Fix TLS-ALPN test that fails when run with newer versions of OpenSSL. More details about these changes can be found on our GitHub repo. diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 4a42a71fd..972bf21df 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1334,7 +1334,9 @@ cryptography==2.8 \ distro==1.4.0 \ --hash=sha256:362dde65d846d23baee4b5c058c8586f219b5a54be1cf5fc6ff55c4578392f57 \ --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4 -enum34==1.1.6 \ +# Package enum34 needs to be explicitly limited to Python2.x, in order to avoid +# certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456. +enum34==1.1.6 ; python_version < '3.4' \ --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \ --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \ --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \ diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index eec5a9946..979d25673 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -78,7 +78,9 @@ cryptography==2.8 \ distro==1.4.0 \ --hash=sha256:362dde65d846d23baee4b5c058c8586f219b5a54be1cf5fc6ff55c4578392f57 \ --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4 -enum34==1.1.6 \ +# Package enum34 needs to be explicitly limited to Python2.x, in order to avoid +# certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456. +enum34==1.1.6 ; python_version < '3.4' \ --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \ --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \ --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \ diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index 204f55d55..338569e6d 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -34,7 +34,7 @@ fi VERSION=$("$PYTHON_NAME" letsencrypt-auto-source/version.py) # setup venv -"$VENV_SCRIPT" --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt +CERTBOT_PIP_NO_BINARY=":all:" "$VENV_SCRIPT" --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt . "$VENV_PATH/bin/activate" # pytest is needed to run tests on some of our packages so we install a pinned version here. tools/pip_install.py pytest diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 5196cf9c4..b1c873918 100644 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -12,6 +12,7 @@ VENV_NAME. from __future__ import print_function +from distutils.version import LooseVersion import glob import os import re @@ -131,6 +132,13 @@ def subprocess_with_print(cmd, env=None, shell=False): subprocess.check_call(cmd, env=env, shell=shell) +def subprocess_output_with_print(cmd, env=None, shell=False): + if env is None: + env = os.environ + print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) + return subprocess.check_output(cmd, env=env, shell=shell) + + def get_venv_python_path(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): @@ -191,9 +199,31 @@ def install_packages(venv_name, pip_args): # Using the python executable from venv, we ensure to execute following commands in this venv. py_venv = get_venv_python_path(venv_name) subprocess_with_print([py_venv, os.path.abspath('letsencrypt-auto-source/pieces/pipstrap.py')]) + # We only use this value during pip install because: + # 1) We're really only adding it for installing cryptography, which happens here, and + # 2) There are issues with calling it along with VIRTUALENV_NO_DOWNLOAD, which applies at the + # steps above, not during pip install. + env_pip_no_binary = os.environ.get('CERTBOT_PIP_NO_BINARY') + if env_pip_no_binary: + # Check OpenSSL version. If it's too low, don't apply the env variable. + openssl_version_string = str(subprocess_output_with_print(['openssl', 'version'])) + matches = re.findall(r'OpenSSL ([^ ]+) ', openssl_version_string) + if not matches: + print('Could not find OpenSSL version, not setting PIP_NO_BINARY.') + else: + openssl_version = matches[0] + + if LooseVersion(openssl_version) >= LooseVersion('1.0.2'): + print('Setting PIP_NO_BINARY to {0}' + ' as specified in CERTBOT_PIP_NO_BINARY'.format(env_pip_no_binary)) + os.environ['PIP_NO_BINARY'] = env_pip_no_binary + else: + print('Not setting PIP_NO_BINARY, as OpenSSL version is too old.') command = [py_venv, os.path.abspath('tools/pip_install.py')] command.extend(pip_args) subprocess_with_print(command) + if 'PIP_NO_BINARY' in os.environ: + del os.environ['PIP_NO_BINARY'] if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific