diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/Dockerfile b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/Dockerfile
index b5a34a4ab..392f5efa6 100644
--- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/Dockerfile
+++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/Dockerfile
@@ -11,6 +11,7 @@ ENV APACHE_RUN_USER=daemon \
     APACHE_LOG_DIR=/usr/local/apache2/logs 
 
 COPY letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh /usr/local/bin/
+COPY letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2dismod.sh /usr/local/bin/
 COPY letsencrypt-compatibility-test/letsencrypt_compatibility_test/testdata/rsa1024_key2.pem /usr/local/apache2/conf/
 COPY letsencrypt-compatibility-test/letsencrypt_compatibility_test/testdata/empty_cert.pem /usr/local/apache2/conf/
 
diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2dismod.sh b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2dismod.sh
new file mode 100755
index 000000000..ca96e216f
--- /dev/null
+++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2dismod.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# An extremely simplified version of `a2enmod` for disabling modules in the
+# httpd docker image. First argument is the server_root and the second is the
+# module to be disabled.
+
+apache_confdir=$1
+module=$2
+
+sed -i "/.*"$module".*/d" "$apache_confdir/test.conf"
+enabled_conf="$apache_confdir/mods-enabled/"$module".conf"
+if [ -e "$enabled_conf" ]
+then
+    rm $enabled_conf
+fi
diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh
index 6c33c0597..f822a1f7b 100755
--- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh
+++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
-# An extremely simplified (and hacky) version of 'a2enmod' for the httpd
-# docker image. First argument is server_root and second argument is the module
-# to be enabled.
+# An extremely simplified version of `a2enmod` for enabling modules in the
+# httpd docker image. First argument is the server_root and the second is the
+# module to be enabled.
 
 APACHE_CONFDIR=$1
 
diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py
index ab14f0ea7..0d3dbb1b5 100644
--- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py
+++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py
@@ -16,7 +16,7 @@ from letsencrypt_compatibility_test.configurators import common as configurators
 
 
 APACHE_VERSION_REGEX = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE)
-APACHE_COMMANDS = ["apachectl", "a2enmod"]
+APACHE_COMMANDS = ["apachectl", "a2enmod", "a2dismod"]
 
 
 class Proxy(configurators_common.Proxy):
@@ -148,7 +148,7 @@ class Proxy(configurators_common.Proxy):
         self.le_config.apache_ctl = "apachectl -d {0} -f {1}".format(
             server_root, config_file)
         self.le_config.apache_enmod = "a2enmod.sh {0}".format(server_root)
-        self.le_config.apache_dismod = self.le_config.apache_enmod
+        self.le_config.apache_dismod = "a2dismod.sh {0}".format(server_root)
         self.le_config.apache_init_script = self.le_config.apache_ctl + " -k"
 
         self._apache_configurator = configurator.ApacheConfigurator(
diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py
index a519e844f..131ef2b5b 100644
--- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py
+++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py
@@ -177,10 +177,14 @@ def test_enhancements(plugin, domains):
     for domain in domains:
         try:
             plugin.enhance(domain, "redirect")
-        except le_errors.Error as error:
+        except le_errors.PluginError as error:
             # Don't immediately fail because a redirect may already be enabled
             logger.warning("Plugin failed to enable redirect for %s:", domain)
             logger.warning("%s", error)
+        except le_errors.Error as error:
+            logger.error("An error occurred while enabling redirect for %s:",
+                         domain)
+            logger.exception(error)
 
     if not _save_and_restart(plugin, "enhanced"):
         return False
@@ -199,13 +203,13 @@ def test_enhancements(plugin, domains):
     return success
 
 
-def _try_until_true(func, max_tries=3):
+def _try_until_true(func, max_tries=5, sleep_time=0.5):
     """Calls func up to max_tries times until it returns True"""
     for _ in xrange(0, max_tries):
         if func():
             return True
         else:
-            time.sleep(1)
+            time.sleep(sleep_time)
 
     return False