diff --git a/CheckSTARTTLS.py b/CheckSTARTTLS.py
index 7c9c94e0f..e5c5a4323 100755
--- a/CheckSTARTTLS.py
+++ b/CheckSTARTTLS.py
@@ -14,6 +14,7 @@ from M2Crypto import X509
 from publicsuffix import PublicSuffixList
 
 public_suffix_list = PublicSuffixList()
+CERTS_OBSERVED = 'certs-observed'
 
 def mkdirp(path):
     try:
@@ -63,7 +64,7 @@ def tls_connect(mx_host, mail_domain):
       return
 
     # Save a copy of the certificate for later analysis
-    with open(os.path.join(mail_domain, mx_host), "w") as f:
+    with open(os.path.join(CERTS_OBSERVED, mail_domain, mx_host), "w") as f:
       f.write(output)
 
 def valid_cert(filename):
@@ -90,9 +91,12 @@ def check_certs(mail_domain):
   Return "" if any certs for any mx domains pointed to by mail_domain
   were invalid, and a public suffix for one if they were all valid
   """
+  dir = os.path.join(CERTS_OBSERVED, mail_domain)
+  if not os.path.exists(dir):
+    collect(mail_domain)
   names = set()
-  for mx_hostname in os.listdir(mail_domain):
-    filename = os.path.join(mail_domain, mx_hostname)
+  for mx_hostname in os.listdir(dir):
+    filename = os.path.join(dir, mx_hostname)
     if not valid_cert(filename):
       return ""
     else:
@@ -131,14 +135,19 @@ def supports_starttls(mx_host):
   except socket.error as e:
     print "Connection to %s failed: %s" % (mx_host, e.strerror)
     return False
-  except smtplib.SMTPException:
-    print "No STARTTLS support on %s" % mx_host
+  except smtplib.SMTPException, e:
+    # In order to talk to some hosts, you need to run this from a host that has a
+    # reverse DNS entry. AWS instances all have reverse DNS, as an example.
+    if e[0] == 554:
+      print e[1]
+    else:
+      print "No STARTTLS support on %s" % mx_host, e[0]
     return False
 
 def min_tls_version(mail_domain):
   protocols = []
-  for mx_hostname in os.listdir(mail_domain):
-    filename = os.path.join(mail_domain, mx_hostname)
+  for mx_hostname in os.listdir(os.path.join(CERTS_OBSERVED, mail_domain)):
+    filename = os.path.join(CERTS_OBSERVED, mail_domain, mx_hostname)
     contents = open(filename).read()
     protocol = re.findall("Protocol  : (.*)", contents)[0]
     protocols.append(protocol)
@@ -151,7 +160,7 @@ def collect(mail_domain):
   subsequent analysis faster.
   """
   print "Checking domain %s" % mail_domain
-  mkdirp(mail_domain)
+  mkdirp(os.path.join(CERTS_OBSERVED, mail_domain))
   answers = dns.resolver.query(mail_domain, 'MX')
   for rdata in answers:
       mx_host = str(rdata.exchange).rstrip(".")
@@ -167,13 +176,9 @@ if __name__ == '__main__':
   for input in sys.argv[1:]:
     for domain in open(input).readlines():
       domain = domain.strip()
-      if not os.path.exists(domain):
-        collect(domain)
-      if len(os.listdir(domain)) == 0:
-        continue
       suffix = check_certs(domain)
-      min_version = min_tls_version(domain)
       if suffix != "":
+        min_version = min_tls_version(domain)
         suffix_match = "." + suffix
         config["acceptable-mxs"][domain] = {
           "accept-mx-domains": [suffix_match]
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 000000000..891e5809d
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,3 @@
+dnspython
+publicsuffix
+m2crypto