Test farm improvements (#5088)

* prevent regressions of #5082

* Fix test_leauto_upgrades.sh

test_leauto_upgrades.sh has been incorrectly been succeeding because while peep
doesn't work with newer versions of pip and letsencrypt-auto would crash,
the output included the version number so we reported the test as passing.
This updates letsencrypt-auto to the oldest version that still works for the
purpose of the test and sets pipefail so errors are properly reported.

* Test symlink creation in test_leauto_upgrades.sh

* Pin dependencies in test_sdists.sh.

* Fix permissions errors in test_tests.sh

tests/letstest/scripts/test_leauto_upgrades.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -xe
+set -o pipefail
 
 # $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL
 # are dynamically set at execution
@@ -15,11 +16,11 @@ if ! command -v git ; then
     fi
 fi
 BRANCH=`git rev-parse --abbrev-ref HEAD`
-# 0.4.1 is the oldest version of letsencrypt-auto that can be used because
-# it's the first version that both pins package versions and properly supports
-# --no-self-upgrade.
-git checkout -f v0.4.1
-if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.4.1 ; then
+# 0.5.0 is the oldest version of letsencrypt-auto that can be used because it's
+# the first version that pins package versions, properly supports
+# --no-self-upgrade, and works with newer versions of pip.
+git checkout -f v0.5.0
+if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then
     echo initial installation appeared to fail
     exit 1
 fi
@@ -31,3 +32,9 @@ if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep $EXPE
     exit 1
 fi
 echo upgrade appeared to be successful
+
+if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then
+    echo symlink from old venv path not properly created!
+    exit 1
+fi
+echo symlink properly created

tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -x
+set -eo pipefail
 
 # $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
 
@@ -36,3 +37,8 @@ if [ "$REVOKED" != 1 ] ; then
     echo "Did not find one revoked cert as expected ($REVOKED)"
     exit 1
 fi
+
+if ! letsencrypt-auto --help | grep -F "letsencrypt-auto [SUBCOMMAND]"; then
+    echo "letsencrypt-auto not included in help output!"
+    exit 1
+fi

tests/letstest/scripts/test_sdists.sh

@@ -13,6 +13,7 @@ virtualenv --no-site-packages -p $PYTHON --setuptools venv
 . ./venv/bin/activate
 pip install -U pip
 pip install -U setuptools
+pip install --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt
 
 # build sdists
 for pkg_dir in acme . $PLUGINS; do

tests/letstest/scripts/test_tests.sh

@@ -4,7 +4,8 @@ MODULES="acme certbot certbot_apache certbot_nginx"
 VENV_NAME=venv
 
 # *-auto respects VENV_PATH
-VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --non-interactive --version
+letsencrypt/certbot-auto --debug --os-packages-only --non-interactive
+LE_AUTO_SUDO="" VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --no-bootstrap --non-interactive --version
 . $VENV_NAME/bin/activate
 
 # change to an empty directory to ensure CWD doesn't affect tests