diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py
index fca2b3e3e..0a0d2634a 100644
--- a/certbot/tests/cli_test.py
+++ b/certbot/tests/cli_test.py
@@ -166,7 +166,6 @@ class ParseTest(unittest.TestCase):
         self.assertTrue("--checkpoints" not in out)
 
         out = self._help_output(['-h'])
-        self.assertTrue("letsencrypt-auto" not in out)  # test cli.cli_command
         if "nginx" in PLUGINS:
             self.assertTrue("Use the Nginx plugin" in out)
         else:
diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh
index ad729e0eb..7c6b1c114 100755
--- a/tests/letstest/scripts/test_sdists.sh
+++ b/tests/letstest/scripts/test_sdists.sh
@@ -44,7 +44,7 @@ for pkg_dir in acme certbot $PLUGINS; do
     cd -
 done
 
-VERSION=$(python letsencrypt-auto-source/version.py)
+VERSION=$(python tests/letstest/scripts/version.py)
 # test sdists
 cd $TEMP_DIR
 for pkg in acme certbot $PLUGINS; do
diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh
deleted file mode 100755
index 858fc1f18..000000000
--- a/tests/letstest/scripts/test_tests.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/sh -xe
-#
-# This script is useful for testing that the packages we've built for a release
-# work on a variety of systems. For an example of the kinds of problems that
-# can occur, see https://github.com/certbot/certbot/issues/3455.
-
-REPO_ROOT="letsencrypt"
-LE_AUTO="$REPO_ROOT/letsencrypt-auto-source/letsencrypt-auto"
-LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive"
-MODULES="acme certbot certbot-apache certbot-nginx"
-PIP_INSTALL="tools/pip_install.py"
-VENV_NAME=venv
-BOOTSTRAP_SCRIPT="$REPO_ROOT/tests/letstest/scripts/bootstrap_os_packages.sh"
-VENV_SCRIPT="tools/venv.py"
-
-sudo $BOOTSTRAP_SCRIPT
-
-cd $REPO_ROOT
-$VENV_SCRIPT
-. $VENV_NAME/bin/activate
-"$PIP_INSTALL" pytest
-
-# To run tests that aren't packaged in modules, run pytest
-# from the repo root. The directory structure should still
-# cause the installed packages to be tested while using
-# the tests available in the subdirectories.
-
-for module in $MODULES ; do
-    echo testing $module
-    pytest -v $module
-done
diff --git a/tests/letstest/scripts/version.py b/tests/letstest/scripts/version.py
new file mode 100644
index 000000000..dedea21dc
--- /dev/null
+++ b/tests/letstest/scripts/version.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+"""Get the current Certbot version number.
+Provides simple utilities for determining the Certbot version number and
+building letsencrypt-auto.
+"""
+from __future__ import print_function
+from os.path import abspath, dirname, join
+import re
+
+
+def certbot_version(build_script_dir):
+    """Return the version number stamped in certbot/__init__.py."""
+    return re.search('''^__version__ = ['"](.+)['"].*''',
+                     file_contents(join(dirname(build_script_dir),
+                                        'certbot',
+                                        'certbot',
+                                        '__init__.py')),
+                     re.M).group(1)
+
+
+def file_contents(path):
+    with open(path) as file:
+        return file.read()
+
+
+if __name__ == '__main__':
+    print(certbot_version(dirname(abspath(__file__))))
diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt
index 4f5eda34e..ae175cdd4 100644
--- a/tools/dev_constraints.txt
+++ b/tools/dev_constraints.txt
@@ -1,5 +1,5 @@
 # Specifies Python package versions for development and building Docker images.
-# It includes in particular packages not specified in letsencrypt-auto's requirements file.
+# It includes in particular packages not specified in tools/certbot_constraints.txt.
 # Some dev package versions specified here may be overridden by higher level constraints
 # files during tests (eg. tools/certbot_constraints.txt).
 alabaster==0.7.10