From be7e99a46193e2b9305fe0f679b4edd178954d6b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 11 May 2017 10:06:05 -0700 Subject: [PATCH] Pin dependency versions when using tools/venv.sh (#4629) * Revert "Pin python-augeas version to avoid error with 1.0.0 (#4422)" This reverts commit 1c51ae25887f2dc3168a38d1f0042363cd7ac1e3. * make dependency-requirements * separate certbot and dependency requirements * fix build.py * update hashin comment * simplify release pinning * separate letsencrypt dependency * pin hashes in venv * error out when bad things happen * use pinned dependencies in tox * Revert "pin hashes in venv" This reverts commit 1cd38a9e50da94e6959728bf57aaf642807bc9c7. * use pip_install.sh in venv_common * quote pip install args * bump mock version --- certbot-apache/setup.py | 2 +- letsencrypt-auto-source/build.py | 7 ++-- letsencrypt-auto-source/letsencrypt-auto | 24 ++++++++------ .../letsencrypt-auto.template | 4 ++- .../pieces/certbot-requirements.txt | 12 +++++++ ...ements.txt => dependency-requirements.txt} | 32 ++++--------------- .../pieces/letsencrypt-requirements.txt | 10 ++++++ tools/_venv_common.sh | 2 +- tools/pip_install.sh | 13 ++++++++ tools/release.sh | 9 ++---- tox.ini | 26 ++++++++------- 11 files changed, 83 insertions(+), 58 deletions(-) create mode 100644 letsencrypt-auto-source/pieces/certbot-requirements.txt rename letsencrypt-auto-source/pieces/{letsencrypt-auto-requirements.txt => dependency-requirements.txt} (88%) create mode 100644 letsencrypt-auto-source/pieces/letsencrypt-requirements.txt create mode 100755 tools/pip_install.sh diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 9429ada2e..ccd7cbc2a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -11,7 +11,7 @@ install_requires = [ 'acme=={0}'.format(version), 'certbot=={0}'.format(version), 'mock', - 'python-augeas<=0.5.0', + 'python-augeas', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', diff --git a/letsencrypt-auto-source/build.py b/letsencrypt-auto-source/build.py index eebad61b7..a1e40fe44 100755 --- a/letsencrypt-auto-source/build.py +++ b/letsencrypt-auto-source/build.py @@ -21,14 +21,17 @@ def build(version=None, requirements=None): :arg version: The version to attach to the script. Default: the version of the certbot package :arg requirements: The contents of the requirements file to embed. Default: - contents of letsencrypt-auto-requirements.txt + contents of dependency-requirements.txt, letsencrypt-requirements.txt, + and certbot-requirements.txt """ special_replacements = { 'LE_AUTO_VERSION': version or certbot_version(DIR) } if requirements: - special_replacements['letsencrypt-auto-requirements.txt'] = requirements + special_replacements['dependency-requirements.txt'] = '' + special_replacements['letsencrypt-requirements.txt'] = '' + special_replacements['certbot-requirements.txt'] = requirements def replacer(match): token = match.group(1) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index d4433d525..7e4a7554f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -694,7 +694,7 @@ if [ "$1" = "--le-auto-phase2" ]; then # Hashin example: # pip install hashin -# hashin -r letsencrypt-auto-requirements.txt cryptography==1.5.2 +# hashin -r dependency-requirements.txt cryptography==1.5.2 # sets the new certbot-auto pinned version of cryptography to 1.5.2 argparse==1.4.0 \ @@ -754,9 +754,9 @@ cryptography==1.5.3 \ enum34==1.1.2 \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 -funcsigs==0.4 \ - --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \ - --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033 +funcsigs==1.0.2 \ + --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ + --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 idna==2.0 \ --hash=sha256:9b2fc50bd3c4ba306b9651b69411ef22026d4d8335b93afc2214cef1246ce707 \ --hash=sha256:16199aad938b290f5be1057c0e1efc6546229391c23cea61ca940c115f7d3d3b @@ -851,15 +851,21 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==1.0.1 \ - --hash=sha256:b839dd2d9c117c701430c149956918a423a9863b48b09c90e30a6013e7d2f44f \ - --hash=sha256:8f83080daa249d036cbccfb8ae5cc6ff007b88d6d937521371afabe7b19badbc +mock==2.0.0 \ + --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ + --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba + +# Contains the requirements for the letsencrypt package. +# +# Since the letsencrypt package depends on certbot and using pip with hashes +# requires that all installed packages have hashes listed, this allows +# dependency-requirements.txt to be used without requiring a hash for a +# (potentially unreleased) Certbot package. + letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -# THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. - acme==0.14.0 \ --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index f6585a378..305435a9b 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -317,7 +317,9 @@ if [ "$1" = "--le-auto-phase2" ]; then # There is no $ interpolation due to quotes on starting heredoc delimiter. # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" -{{ letsencrypt-auto-requirements.txt }} +{{ dependency-requirements.txt }} +{{ letsencrypt-requirements.txt }} +{{ certbot-requirements.txt }} UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt new file mode 100644 index 000000000..a49abeefd --- /dev/null +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -0,0 +1,12 @@ +acme==0.14.0 \ + --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ + --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 +certbot==0.14.0 \ + --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ + --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 +certbot-apache==0.14.0 \ + --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ + --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f +certbot-nginx==0.14.0 \ + --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ + --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt similarity index 88% rename from letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt rename to letsencrypt-auto-source/pieces/dependency-requirements.txt index 5db66fe88..4cba83195 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -5,7 +5,7 @@ # Hashin example: # pip install hashin -# hashin -r letsencrypt-auto-requirements.txt cryptography==1.5.2 +# hashin -r dependency-requirements.txt cryptography==1.5.2 # sets the new certbot-auto pinned version of cryptography to 1.5.2 argparse==1.4.0 \ @@ -65,9 +65,9 @@ cryptography==1.5.3 \ enum34==1.1.2 \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 -funcsigs==0.4 \ - --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \ - --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033 +funcsigs==1.0.2 \ + --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ + --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 idna==2.0 \ --hash=sha256:9b2fc50bd3c4ba306b9651b69411ef22026d4d8335b93afc2214cef1246ce707 \ --hash=sha256:16199aad938b290f5be1057c0e1efc6546229391c23cea61ca940c115f7d3d3b @@ -162,24 +162,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==1.0.1 \ - --hash=sha256:b839dd2d9c117c701430c149956918a423a9863b48b09c90e30a6013e7d2f44f \ - --hash=sha256:8f83080daa249d036cbccfb8ae5cc6ff007b88d6d937521371afabe7b19badbc -letsencrypt==0.7.0 \ - --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ - --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 - -# THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. - -acme==0.14.0 \ - --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ - --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 -certbot==0.14.0 \ - --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ - --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 -certbot-apache==0.14.0 \ - --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ - --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f -certbot-nginx==0.14.0 \ - --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ - --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db +mock==2.0.0 \ + --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ + --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba diff --git a/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt new file mode 100644 index 000000000..8e745c9cd --- /dev/null +++ b/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt @@ -0,0 +1,10 @@ +# Contains the requirements for the letsencrypt package. +# +# Since the letsencrypt package depends on certbot and using pip with hashes +# requires that all installed packages have hashes listed, this allows +# dependency-requirements.txt to be used without requiring a hash for a +# (potentially unreleased) Certbot package. + +letsencrypt==0.7.0 \ + --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ + --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh index ddbb02c62..20ed4c034 100755 --- a/tools/_venv_common.sh +++ b/tools/_venv_common.sh @@ -19,7 +19,7 @@ virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS # invocations use latest pip install -U pip pip install -U setuptools -pip install "$@" +./tools/pip_install.sh "$@" set +x echo "Please run the following command to activate developer environment:" diff --git a/tools/pip_install.sh b/tools/pip_install.sh new file mode 100755 index 000000000..8a58f9e48 --- /dev/null +++ b/tools/pip_install.sh @@ -0,0 +1,13 @@ +#!/bin/sh -e +# pip installs packages using Certbot's requirements file as constraints + +# get the root of the Certbot repo +repo_root=$(git rev-parse --show-toplevel) +requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" +constraints=$(mktemp) +trap "rm -f $constraints" EXIT +# extract pinned requirements without hashes +sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $constraints + +# install the requested packages using the pinned requirements as constraints +pip install --constraint $constraints "$@" diff --git a/tools/release.sh b/tools/release.sh index 1da11fe2c..3b1d4d8a6 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -170,19 +170,14 @@ cd ~- for pkg in acme certbot certbot-apache certbot-nginx ; do echo $pkg==$version \\ pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' -done > /tmp/hashes.$$ +done > letsencrypt-auto-source/pieces/certbot-requirements.txt deactivate -if ! wc -l /tmp/hashes.$$ | grep -qE "^\s*12 " ; then +if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*12 " ; then echo Unexpected pip hash output exit 1 fi -# perform hideous surgery on requirements.txt... -head -n -12 letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt > /tmp/req.$$ -cat /tmp/hashes.$$ >> /tmp/req.$$ -cp /tmp/req.$$ letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt - # ensure we have the latest built version of leauto letsencrypt-auto-source/build.py diff --git a/tox.ini b/tox.ini index a956b19a9..771b92d11 100644 --- a/tox.ini +++ b/tox.ini @@ -10,20 +10,22 @@ envlist = modification,py{26,33,34,35,36},cover,lint # loops, especially on Travis [base] +# wraps pip install to use pinned versions of dependencies +pip_install = {toxinidir}/tools/pip_install.sh # packages installed separately to ensure that downstream deps problems # are detected, c.f. #1002 core_commands = - pip install -e acme[dev] + {[base]pip_install} -e acme[dev] nosetests -v acme --processes=-1 - pip install -e .[dev] + {[base]pip_install} -e .[dev] nosetests -v certbot --processes=-1 --process-timeout=100 core_install_args = -e acme[dev] -e .[dev] core_paths = acme/acme certbot plugin_commands = - pip install -e certbot-apache + {[base]pip_install} -e certbot-apache nosetests -v certbot_apache --processes=-1 --process-timeout=80 - pip install -e certbot-nginx + {[base]pip_install} -e certbot-nginx nosetests -v certbot_nginx --processes=-1 plugin_install_args = -e certbot-apache -e certbot-nginx plugin_paths = certbot-apache/certbot_apache certbot-nginx/certbot_nginx @@ -38,7 +40,7 @@ compatibility_install_args = -e certbot-compatibility-test compatibility_paths = certbot-compatibility-test/certbot_compatibility_test other_commands = - pip install -e letshelp-certbot + {[base]pip_install} -e letshelp-certbot nosetests -v letshelp_certbot --processes=-1 python tests/lock_test.py other_install_args = -e letshelp-certbot @@ -69,12 +71,12 @@ deps = [testenv:py27_install] basepython = python2.7 commands = - pip install {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} + {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} [testenv:cover] basepython = python2.7 commands = - pip install {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} + {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} ./tox.cover.sh [testenv:lint] @@ -84,25 +86,25 @@ basepython = python2.7 # duplicate code checking; if one of the commands fails, others will # continue, but tox return code will reflect previous error commands = - pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + {[base]pip_install} -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} pylint --reports=n --rcfile=.pylintrc {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths} [testenv:mypy] basepython = python3.4 commands = - pip install mypy - pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + {[base]pip_install} mypy + {[base]pip_install} -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} mypy --py2 --ignore-missing-imports {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths} [testenv:apacheconftest] #basepython = python2.7 commands = - pip install {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} {toxinidir}/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test --debian-modules [testenv:nginxroundtrip] commands = - pip install {[base]core_install_args} -e certbot-nginx + {[base]pip_install} {[base]core_install_args} -e certbot-nginx python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata # This is a duplication of the command line in testenv:le_auto to