diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index 9218bdc52..8cc90b64e 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -19,7 +19,7 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share}
 VENV_NAME="letsencrypt"
 VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"}
 VENV_BIN=${VENV_PATH}/bin
-LE_AUTO_VERSION="0.4.0"
+LE_AUTO_VERSION="0.5.0.dev0"
 
 # This script takes the same arguments as the main letsencrypt program, but it
 # additionally responds to --verbose (more output) and --debug (allow support
@@ -1630,6 +1630,7 @@ UNLIKELY_EOF
       # Report error. (Otherwise, be quiet.)
       echo "Had a problem while downloading and verifying Python packages:"
       echo "$PEEP_OUT"
+      rm -rf "$VENV_PATH"
       exit 1
     fi
   fi
diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template
index ad8c97a7f..53838280b 100755
--- a/letsencrypt-auto-source/letsencrypt-auto.template
+++ b/letsencrypt-auto-source/letsencrypt-auto.template
@@ -207,6 +207,7 @@ UNLIKELY_EOF
       # Report error. (Otherwise, be quiet.)
       echo "Had a problem while downloading and verifying Python packages:"
       echo "$PEEP_OUT"
+      rm -rf "$VENV_PATH"
       exit 1
     fi
   fi
diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py
index 32d591190..90e09f57f 100644
--- a/letsencrypt-auto-source/tests/auto_test.py
+++ b/letsencrypt-auto-source/tests/auto_test.py
@@ -5,7 +5,7 @@ from contextlib import contextmanager
 from functools import partial
 from json import dumps
 from os import chmod, environ
-from os.path import abspath, dirname, join
+from os.path import abspath, dirname, exists, join
 import re
 from shutil import copy, rmtree
 import socket
@@ -338,6 +338,12 @@ class AutoTests(TestCase):
                     self.assertIn("THE FOLLOWING PACKAGES DIDN'T MATCH THE "
                                   "HASHES SPECIFIED IN THE REQUIREMENTS",
                                   exc.output)
+                    ok_(not exists(join(venv_dir, 'letsencrypt')),
+                        msg="The virtualenv was left around, even though "
+                            "installation didn't succeed. We shouldn't do "
+                            "this, as it foils our detection of whether we "
+                            "need to recreate the virtualenv, which hinges "
+                            "on the presence of $VENV_BIN/letsencrypt.")
                 else:
                     self.fail("Peep didn't detect a bad hash and stop the "
                               "installation.")