diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py
index 691688969..3d48d100f 100644
--- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py
+++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py
@@ -310,10 +310,10 @@ class NginxConfigurator(common.Plugin):
         key = OpenSSL.crypto.load_privatekey(
             OpenSSL.crypto.FILETYPE_PEM, le_key.pem)
         cert = acme_crypto_util.gen_ss_cert(key, domains=[socket.gethostname()])
-        cert_path = os.path.join(tmp_dir, "cert.pem")
         cert_pem = OpenSSL.crypto.dump_certificate(
             OpenSSL.crypto.FILETYPE_PEM, cert)
-        with open(cert_path, 'w') as cert_file:
+        cert_file, cert_path = le_util.unique_file(os.path.join(tmp_dir, "cert.pem"))
+        with cert_file:
             cert_file.write(cert_pem)
         return cert_path, le_key.file
 
diff --git a/letsencrypt/crypto_util.py b/letsencrypt/crypto_util.py
index 730c32398..76265a739 100644
--- a/letsencrypt/crypto_util.py
+++ b/letsencrypt/crypto_util.py
@@ -53,8 +53,8 @@ def init_save_key(key_size, key_dir, keyname="key-letsencrypt.pem"):
                                config.strict_permissions)
     key_f, key_path = le_util.unique_file(
         os.path.join(key_dir, keyname), 0o600)
-    key_f.write(key_pem)
-    key_f.close()
+    with key_f:
+        key_f.write(key_pem)
 
     logger.info("Generating key (%d bits): %s", key_size, key_path)