diff --git a/tests/letstest/scripts/bootstrap_os_packages.sh b/tests/letstest/scripts/bootstrap_os_packages.sh
index 9e827913c..3adc2810c 100755
--- a/tests/letstest/scripts/bootstrap_os_packages.sh
+++ b/tests/letstest/scripts/bootstrap_os_packages.sh
@@ -46,6 +46,7 @@ BootstrapDebCommon() {
     openssl \
     libffi-dev \
     ca-certificates \
+    make # needed on debian 9 arm64 which doesn't have a python3 pynacl wheel
 
 }
 
diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh
index 078b3f52c..82c7fe1f2 100755
--- a/tests/letstest/scripts/test_apache2.sh
+++ b/tests/letstest/scripts/test_apache2.sh
@@ -40,14 +40,11 @@ fi
 cd letsencrypt
 
 echo "Bootstrapping dependencies..."
-sudo tests/letstest/scripts/bootstrap_os_packages.sh
+sudo tests/letstest/scripts/bootstrap_os_packages.sh . "tools/venv3.py -e acme[dev] -e certbot[dev,docs] -e certbot-apache"
 if [ $? -ne 0 ] ; then
     exit 1
 fi
 
-. tests/letstest/scripts/set_python_envvars.sh
-
-tools/venv3.py -e acme[dev] -e certbot[dev,docs] -e certbot-apache
 sudo "venv3/bin/certbot" -v --debug --text --agree-tos \
                    --renew-by-default --redirect --register-unsafely-without-email \
                    --domain $PUBLIC_HOSTNAME --server $BOULDER_URL