diff --git a/.gitignore b/.gitignore
index a50dddc06..285e68a42 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,12 +30,6 @@ tags
 # auth --cert-path --chain-path
 /*.pem
 
-# letstest
-tests/letstest/letest-*/
-tests/letstest/*.pem
-tests/letstest/venv/
-tests/letstest/venv3/
-
 .venv
 
 # pytest cache
diff --git a/tests/letstest/README.md b/letstest/README.md
similarity index 77%
rename from tests/letstest/README.md
rename to letstest/README.md
index aea4e2e13..2bb0b1788 100644
--- a/tests/letstest/README.md
+++ b/letstest/README.md
@@ -14,15 +14,10 @@ Simple AWS testfarm scripts for certbot client testing
     are needed, they need to be requested via online webform.
 
 ## Installation and configuration
-These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you're
-on a Debian based system, make sure you also have the python3-venv package
-installed. If you have Python 3 installed, you can use requirements.txt to
-create a virtual environment with a known set of dependencies by running:
-```
-python3 -m venv venv3
-. ./venv3/bin/activate
-pip install --requirement requirements.txt
-```
+
+This package is installed in the Certbot development environment that is
+created by following the instructions at
+https://certbot.eff.org/docs/contributing.html#running-a-local-copy-of-the-client.
 
 You can then configure AWS credentials and create a key by running:
 ```
@@ -35,9 +30,9 @@ Note: whatever you pick for `<key name>` will be shown to other users with AWS a
 When prompted for a default region name, enter: `us-east-1`.
 
 ## Usage
-To run tests, activate the virtual environment you created above and run:
+To run tests, activate the virtual environment you created above and from this directory run:
 ```
->python multitester.py targets.yaml /path/to/your/key.pem <profile name> scripts/<test to run>
+>letstest targets/targets.yaml /path/to/your/key.pem <profile name> scripts/<test to run>
 ```
 
 You can only run up to two tests at once. The following error is often indicative of there being too many AWS instances running on our account:
@@ -52,7 +47,7 @@ aws ec2 terminate-instances --profile <profile name> --instance-ids $(aws ec2 de
 
 It will take a minute for these instances to shut down and become available again. Running this will invalidate any in progress tests.
 
-A folder named `letest-<timestamp>` is also created with a log file from each instance of the test and a file named "results" containing the output above.
+A temporary directory whose name is output by the tests is also created with a log file from each instance of the test and a file named "results" containing the output above.
 The tests take quite a while to run.
 
 ## Scripts
diff --git a/tests/letstest/multitester.py b/letstest/letstest/multitester.py
similarity index 98%
rename from tests/letstest/multitester.py
rename to letstest/letstest/multitester.py
index 360bb979e..a56bf0f37 100644
--- a/tests/letstest/multitester.py
+++ b/letstest/letstest/multitester.py
@@ -22,7 +22,7 @@ Usage:
 >aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \
  --query 'KeyMaterial' --output text > MyKeyPair.pem
 then:
->python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_sdists.sh
+>letstest targets/targets.yaml MyKeyPair.pem HappyHacker scripts/test_sdists.sh
 see:
   https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
   https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
@@ -33,6 +33,7 @@ from multiprocessing import Manager
 import os
 import socket
 import sys
+import tempfile
 import time
 import traceback
 import urllib.error as urllib_error
@@ -374,9 +375,8 @@ def main():
 
     # Set up local copy of git repo
     #-------------------------------------------------------------------------------
-    log_dir = "letest-%d"%int(time.time()) #points to logging / working directory
-    print("Making local dir for test repo and logs: %s"%log_dir)
-    local_cxn.local('mkdir %s'%log_dir)
+    log_dir = tempfile.mkdtemp()  # points to logging / working directory
+    print("Local dir for test repo and logs: %s"%log_dir)
 
     try:
         # figure out what git object to test and locally create it in log_dir
diff --git a/tests/letstest/scripts/bootstrap_os_packages.sh b/letstest/scripts/bootstrap_os_packages.sh
similarity index 100%
rename from tests/letstest/scripts/bootstrap_os_packages.sh
rename to letstest/scripts/bootstrap_os_packages.sh
diff --git a/tests/letstest/scripts/test_apache2.sh b/letstest/scripts/test_apache2.sh
similarity index 96%
rename from tests/letstest/scripts/test_apache2.sh
rename to letstest/scripts/test_apache2.sh
index 77dc35f1e..9d9ca6c12 100755
--- a/tests/letstest/scripts/test_apache2.sh
+++ b/letstest/scripts/test_apache2.sh
@@ -59,7 +59,7 @@ fi
 cd letsencrypt
 
 echo "Bootstrapping dependencies..."
-sudo tests/letstest/scripts/bootstrap_os_packages.sh
+sudo letstest/scripts/bootstrap_os_packages.sh
 if [ $? -ne 0 ] ; then
     exit 1
 fi
@@ -113,7 +113,7 @@ elif [ "$OS_TYPE" = "centos" ]; then
 fi
 OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9)
 APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8)
-"venv/bin/python" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION"
+"venv/bin/python" letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION"
 if [ $? -ne 0 ] ; then
     FAIL=1
 fi
diff --git a/tests/letstest/scripts/test_openssl_version.py b/letstest/scripts/test_openssl_version.py
similarity index 100%
rename from tests/letstest/scripts/test_openssl_version.py
rename to letstest/scripts/test_openssl_version.py
diff --git a/tests/letstest/scripts/test_sdists.sh b/letstest/scripts/test_sdists.sh
similarity index 92%
rename from tests/letstest/scripts/test_sdists.sh
rename to letstest/scripts/test_sdists.sh
index b0d53fb67..562169524 100755
--- a/tests/letstest/scripts/test_sdists.sh
+++ b/letstest/scripts/test_sdists.sh
@@ -2,7 +2,7 @@
 
 cd letsencrypt
 
-BOOTSTRAP_SCRIPT="tests/letstest/scripts/bootstrap_os_packages.sh"
+BOOTSTRAP_SCRIPT="letstest/scripts/bootstrap_os_packages.sh"
 VENV_PATH=venv
 
 # install OS packages
@@ -41,7 +41,7 @@ for pkg_dir in acme certbot $PLUGINS; do
     cd -
 done
 
-VERSION=$(python tests/letstest/scripts/version.py)
+VERSION=$(python letstest/scripts/version.py)
 # test sdists
 cd $TEMP_DIR
 for pkg in acme certbot $PLUGINS; do
diff --git a/letstest/setup.py b/letstest/setup.py
new file mode 100644
index 000000000..ba0b54efb
--- /dev/null
+++ b/letstest/setup.py
@@ -0,0 +1,45 @@
+from setuptools import find_packages
+from setuptools import setup
+
+setup(
+    name='letstest',
+    version='1.0',
+    description='Test Certbot on different AWS images',
+    url='https://github.com/certbot/certbot',
+    author='Certbot Project',
+    author_email='certbot-dev@eff.org',
+    license='Apache License 2.0',
+    python_requires='>=3.6',
+    classifiers=[
+        'Development Status :: 5 - Production/Stable',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: Apache Software License',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Topic :: Internet :: WWW/HTTP',
+        'Topic :: Security',
+    ],
+
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=[
+        # This isn't required by the tests themselves, but it is a useful tool
+        # to have when using these tests to generate keys and control running
+        # instances so the dependency is declared here for convenience.
+        'awscli',
+        'boto3',
+        'botocore',
+        # The API from Fabric 2.0+ is used instead of the 1.0 API.
+        'fabric>=2',
+        'pyyaml',
+    ],
+    entry_points={
+        'console_scripts': [
+            'letstest=letstest.multitester',
+        ],
+    }
+)
diff --git a/tests/letstest/apache2_targets.yaml b/letstest/targets/apache2_targets.yaml
similarity index 100%
rename from tests/letstest/apache2_targets.yaml
rename to letstest/targets/apache2_targets.yaml
diff --git a/tests/letstest/scripts/version.py b/letstest/targets/scripts/version.py
similarity index 100%
rename from tests/letstest/scripts/version.py
rename to letstest/targets/scripts/version.py
diff --git a/tests/letstest/targets.yaml b/letstest/targets/targets.yaml
similarity index 100%
rename from tests/letstest/targets.yaml
rename to letstest/targets/targets.yaml
diff --git a/tests/letstest/auto_targets.yaml b/tests/letstest/auto_targets.yaml
deleted file mode 100644
index 01d410227..000000000
--- a/tests/letstest/auto_targets.yaml
+++ /dev/null
@@ -1,59 +0,0 @@
-# These images are located in us-east-1.
-
-targets:
-  #-----------------------------------------------------------------------------
-  #Ubuntu
-  - ami: ami-095192256fe1477ad
-    name: ubuntu18.04LTS
-    type: ubuntu
-    virt: hvm
-    user: ubuntu
-  - ami: ami-09677e0a6b14905b0
-    name: ubuntu16.04LTS
-    type: ubuntu
-    virt: hvm
-    user: ubuntu
-  #-----------------------------------------------------------------------------
-  # Debian
-  - ami: ami-01db78123b2b99496
-    name: debian10
-    type: ubuntu
-    virt: hvm
-    user: admin
-  - ami: ami-003f19e0e687de1cd
-    name: debian9
-    type: ubuntu
-    virt: hvm
-    user: admin
-  - ami: ami-0ed54dd1b25657636
-    name: debian9_arm64
-    type: ubuntu
-    virt: hvm
-    user: admin
-    machine_type: a1.medium
-  #-----------------------------------------------------------------------------
-  # Other Redhat Distros
-  - ami: ami-0916c408cb02e310b
-    name: RHEL7
-    type: centos
-    virt: hvm
-    user: ec2-user
-  - ami: ami-0c322300a1dd5dc79
-    name: RHEL8
-    type: centos
-    virt: hvm
-    user: ec2-user
-  #-----------------------------------------------------------------------------
-  # CentOS
-  # These Marketplace AMIs must, irritatingly, have their terms manually
-  # agreed to on the AWS marketplace site for any new AWS account using them...
-  - ami: ami-9887c6e7
-    name: centos7
-    type: centos
-    virt: hvm
-    user: centos
-  - ami: ami-01ca03df4a6012157
-    name: centos8
-    type: centos
-    virt: hvm
-    user: centos
diff --git a/tests/letstest/requirements.txt b/tests/letstest/requirements.txt
deleted file mode 100644
index b49489283..000000000
--- a/tests/letstest/requirements.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-awscli==1.19.36
-bcrypt==3.2.0
-boto3==1.17.36
-botocore==1.20.36
-cffi==1.14.5
-colorama==0.4.3
-cryptography==3.4.6
-docutils==0.15.2
-fabric==2.6.0
-invoke==1.5.0
-jmespath==0.10.0
-paramiko==2.7.2
-pathlib2==2.3.5
-pyasn1==0.4.8
-pycparser==2.20
-PyNaCl==1.4.0
-python-dateutil==2.8.1
-PyYAML==5.4.1
-rsa==4.5
-s3transfer==0.3.6
-six==1.15.0
-urllib3==1.26.4
diff --git a/tools/_release.sh b/tools/_release.sh
index ba54536bf..4e118c2d7 100755
--- a/tools/_release.sh
+++ b/tools/_release.sh
@@ -211,8 +211,8 @@ git add certbot/CHANGELOG.md
 git commit -m "Add contents to certbot/CHANGELOG.md for next version"
 
 echo "New root: $root"
-echo "Test commands (in the letstest repo):"
-echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh'
+echo "Test commands (in the letstest directory):"
+echo 'letstest --saveinstances targets/targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh'
 echo "In order to upload packages run the following command:"
 echo twine upload "$root/dist.$version/*/*"
 
diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml
index 34fea18a6..27e2cab7f 100644
--- a/tools/pinning/pyproject.toml
+++ b/tools/pinning/pyproject.toml
@@ -32,9 +32,18 @@ certbot-nginx = {path = "../../certbot-nginx", extras = ["docs"]}
 certbot-apache = {path = "../../certbot-apache", extras = ["dev"]}
 certbot = {path = "../../certbot", extras = ["dev", "docs"]}
 acme = {path = "../../acme", extras = ["dev", "docs"]}
+letstest = {path = "../../letstest"}
 windows-installer = {path = "../../windows-installer"}
 
 # Extra dependencies
+# awscli is just listed here as a performance optimization. As of writing this,
+# there are some conflicts in shared dependencies between it and other packages
+# we depend on. To try and resolve them, poetry searches through older versions
+# of awscli to see if that resolves the conflict, but there are over 1000
+# versions of awscli on PyPI so this process takes too long. Providing a high
+# minimum version here prevents poetry from searching this path and it fairly
+# quickly resolves the dependency conflict in another way.
+awscli = "^1.19.62"
 # As of writing this, cython is a build dependency of pyyaml. Since there
 # doesn't appear to be a good way to automatically track down and pin build
 # dependencies in Python (see
diff --git a/tools/requirements.txt b/tools/requirements.txt
index b662744c6..4fb1f87e6 100644
--- a/tools/requirements.txt
+++ b/tools/requirements.txt
@@ -12,25 +12,26 @@ appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or pyt
 appnope==0.1.2
 astroid==2.3.3; python_version >= "3.6"
 attrs==20.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+awscli==1.19.62; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
 azure-devops==6.0.0b4; python_version >= "3.6"
-babel==2.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
+babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 backcall==0.2.0
-bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+bcrypt==3.2.0; python_version >= "3.6"
 beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0"
 bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-boto3==1.17.53; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-botocore==1.20.53; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+boto3==1.17.62; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+botocore==1.20.62; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
+cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 cachy==0.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 cleo==0.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 clikit==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 cloudflare==2.8.15; python_version >= "3.6"
-colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+colorama==0.4.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 configargparse==1.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 configobj==5.0.6; python_version >= "3.6"
 coverage==4.5.4; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0" and python_version < "4")
@@ -47,14 +48,15 @@ docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0
 docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+docutils==0.15.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
 execnet==1.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+fabric==2.6.0; python_version >= "3.6"
 filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0"
 future==0.18.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0"
 google-api-core==1.26.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-google-api-python-client==2.2.0; python_version >= "3.6"
+google-api-python-client==2.3.0; python_version >= "3.6"
 google-auth-httplib2==0.1.0; python_version >= "3.6"
-google-auth==1.29.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+google-auth==1.30.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 httplib2==0.19.1; python_version >= "3.6"
@@ -62,6 +64,7 @@ idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" and python
 imagesize==1.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0")
 importlib-resources==5.1.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7"
+invoke==1.5.0; python_version >= "3.6"
 ipdb==0.13.7; python_version >= "3.6"
 ipython-genutils==0.2.0; python_version == "3.6"
 ipython==7.16.1; python_version == "3.6"
@@ -93,6 +96,7 @@ paramiko==2.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or py
 parsedatetime==2.6; python_version >= "3.6"
 parso==0.8.2; python_version == "3.6"
 pastel==0.2.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+pathlib2==2.3.5; python_version >= "3.6"
 pexpect==4.8.0
 pickleshare==0.7.5
 pkginfo==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
@@ -105,14 +109,14 @@ protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or p
 ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-pyasn1==0.4.8; python_version >= "3.6" and python_version < "4"
+pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-pygithub==1.54.1; python_version >= "3.6"
+pygithub==1.55; python_version >= "3.6"
 pygments==2.8.1
-pyjwt==1.7.1; python_version >= "3.6"
+pyjwt==2.1.0; python_version >= "3.6"
 pylev==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 pylint==2.4.3; python_version >= "3.5"
-pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 pynsist==2.7; python_version >= "3.6"
 pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
@@ -126,7 +130,7 @@ pytest==3.2.5
 python-augeas==0.5.0
 python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 python-digitalocean==1.16.0; python_version >= "3.6"
-python-dotenv==0.17.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+python-dotenv==0.17.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0"
 pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "win32"
 pywin32==300; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6")
@@ -140,10 +144,10 @@ requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0
 requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.6.0" and python_version < "4.0"
 rfc3986==1.4.0; python_version >= "3.6"
 rsa==4.7.2; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
-s3transfer==0.3.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+s3transfer==0.4.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux"
 shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+six==1.15.0; python_version == "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version == "3.6"
 snowballstemmer==2.1.0; python_version >= "3.6"
 soupsieve==2.2.1; python_version >= "3.6"
 sphinx-rtd-theme==0.5.2; python_version >= "3.6"
@@ -166,7 +170,7 @@ typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" an
 typing-extensions==3.7.4.3; python_version >= "3.6"
 uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 urllib3==1.26.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
-virtualenv==20.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+virtualenv==20.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 wcwidth==0.2.5; python_version == "3.6"
 webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 websocket-client==0.58.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
diff --git a/tools/venv.py b/tools/venv.py
index 2e0607e70..f3f5781fa 100755
--- a/tools/venv.py
+++ b/tools/venv.py
@@ -44,6 +44,7 @@ REQUIREMENTS = [
     '-e certbot-nginx',
     '-e certbot-compatibility-test',
     '-e certbot-ci',
+    '-e letstest',
 ]
 
 if sys.platform == 'win32':
diff --git a/tox.ini b/tox.ini
index f7db07b52..28a589b1a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -277,22 +277,26 @@ passenv = DOCKER_*
 setenv = {[testenv:oldest]setenv}
 
 [testenv:test-farm-tests-base]
-changedir = tests/letstest
-deps = -rtests/letstest/requirements.txt
+changedir = letstest
+commands = {[base]pip_install} letstest
 passenv =
     AWS_*
 setenv = AWS_DEFAULT_REGION=us-east-1
 
 [testenv:test-farm-apache2]
 changedir = {[testenv:test-farm-tests-base]changedir}
-commands = {toxinidir}/tools/retry.sh python multitester.py apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
+commands =
+    {[testenv:test-farm-tests-base]commands}
+    {toxinidir}/tools/retry.sh letstest targets/apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
 deps = {[testenv:test-farm-tests-base]deps}
 passenv = {[testenv:test-farm-tests-base]passenv}
 setenv = {[testenv:test-farm-tests-base]setenv}
 
 [testenv:test-farm-sdists]
 changedir = {[testenv:test-farm-tests-base]changedir}
-commands = {toxinidir}/tools/retry.sh python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir}
+commands =
+    {[testenv:test-farm-tests-base]commands}
+    {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir}
 deps = {[testenv:test-farm-tests-base]deps}
 passenv = {[testenv:test-farm-tests-base]passenv}
 setenv = {[testenv:test-farm-tests-base]setenv}