diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index 0630c649a..b542c8d3e 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -1640,6 +1640,7 @@ UNLIKELY_EOF
       # Report error. (Otherwise, be quiet.)
       echo "Had a problem while downloading and verifying Python packages:"
       echo "$PEEP_OUT"
+      rm -rf "$VENV_PATH"
       exit 1
     fi
     echo "Installation succeeded."
diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template
index 22dfba1b2..bccd9e2c9 100755
--- a/letsencrypt-auto-source/letsencrypt-auto.template
+++ b/letsencrypt-auto-source/letsencrypt-auto.template
@@ -210,6 +210,7 @@ UNLIKELY_EOF
       # Report error. (Otherwise, be quiet.)
       echo "Had a problem while downloading and verifying Python packages:"
       echo "$PEEP_OUT"
+      rm -rf "$VENV_PATH"
       exit 1
     fi
     echo "Installation succeeded."
diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py
index 32d591190..90e09f57f 100644
--- a/letsencrypt-auto-source/tests/auto_test.py
+++ b/letsencrypt-auto-source/tests/auto_test.py
@@ -5,7 +5,7 @@ from contextlib import contextmanager
 from functools import partial
 from json import dumps
 from os import chmod, environ
-from os.path import abspath, dirname, join
+from os.path import abspath, dirname, exists, join
 import re
 from shutil import copy, rmtree
 import socket
@@ -338,6 +338,12 @@ class AutoTests(TestCase):
                     self.assertIn("THE FOLLOWING PACKAGES DIDN'T MATCH THE "
                                   "HASHES SPECIFIED IN THE REQUIREMENTS",
                                   exc.output)
+                    ok_(not exists(join(venv_dir, 'letsencrypt')),
+                        msg="The virtualenv was left around, even though "
+                            "installation didn't succeed. We shouldn't do "
+                            "this, as it foils our detection of whether we "
+                            "need to recreate the virtualenv, which hinges "
+                            "on the presence of $VENV_BIN/letsencrypt.")
                 else:
                     self.fail("Peep didn't detect a bad hash and stop the "
                               "installation.")