diff --git a/letsencrypt/revoker.py b/letsencrypt/revoker.py
index 7fe1bd106..32c6f003d 100644
--- a/letsencrypt/revoker.py
+++ b/letsencrypt/revoker.py
@@ -308,7 +308,7 @@ class Revoker(object):
                 "Did not find all cert_list items to remove from LIST")
 
         shutil.copy2(list_path2, self.list_path)
-        newfile.close()
+        os.remove(list_path2)
 
     def _row_to_backup(self, row):
         """Convenience function