Merge branch 'log-summary'

Including adding dmwilcox's mandatory policy argument to jsha's argparse code

ConfigParser.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+import sys
+import json
+from datetime import datetime
+import string
+import collections
+
+def parse_timestamp(ts):
+  try:
+    int(ts)
+    dt = datetime.fromtimestamp(ts)
+    return dt
+  except:
+    raise ValueError, "Invalid timestamp integer: " + `ts`
+
+legal = string.letters + string.digits + ".-"
+known_tlds =["com","org","net","biz","info",] # xxx make me from an ICANN list
+def looks_like_a_domain(s):
+  "Return true if string looks like a domain, as best we can tell..."
+  global known_tlds
+  try:
+    domain = s.lower()
+    assert domain[0].islower()
+    assert all([c in legal for c in domain])
+    tld = s.split(".")[-1]
+    if tld not in known_tlds:
+      # XXX perform DNS query to determine that this TLD exists
+      pass
+    return True
+  except:
+    return False
+
+class Config:
+  def __init__(self, cfg_file_name = "config.json"):
+    f = open(cfg_file_name)
+    self.cfg = json.loads(f.read())
+    self.tls_policies = {}
+    self.mx_map = {}
+    for atr, val in self.cfg.items():
+      # Verify each attribute of the structure
+      if atr.startswith("comment"):
+        continue
+      if atr == "author":
+        if type(val) not in [str, unicode]:
+          raise TypeError, "Author must be a string: " + `val`
+      elif atr == "timestamp":
+        self.timestamp = parse_timestamp(val)
+      elif atr == "expires":
+        self.expires = parse_timestamp(val)
+      elif atr == "tls-policies":
+        for domain, policies in self.check_tls_policy_domains(val):
+          if type(policies) != dict:
+            raise TypeError, domain + "'s policies should be a dict: " + `policies`
+          self.tls_policies[domain] = {} # being here enforces TLS at all
+          for policy, v in policies.items():
+            value = str(v).lower()
+            if policy == "require-tls":
+              if value in ("true", "1", "yes"):
+                self.tls_policies[domain]["required"] = True
+              elif value in ("false", "0", "no"):
+                self.tls_policies[domain]["required"] = False
+              else:
+                raise ValueError, "Unknown require-tls value " + `value`
+            elif policy == "min-tls-version":
+              reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"]
+              reasonable = map(string.lower, reasonable)
+              if not value in reasonable:
+                raise ValueError, "Not a valid TLS version string: " + `value`
+              self.tls_policies[domain]["min-tls-version"] = str(value)
+            elif policy == "enforce-mode":
+              if value == "enforce":
+                self.tls_policies[domain]["enforce"] = True
+              elif value == "log-only":
+                self.tls_policies[domain]["enforce"] = False
+              else:
+                raise ValueError, "Not a known enoforcement policy " + `value`
+      elif atr == "acceptable-mxs":
+        self.acceptable_mxs = val
+        self.mx_domain_to_address_domains = collections.defaultdict(set)
+        for address_domain, properties in self.acceptable_mxs.items():
+          mx_list = properties["accept-mx-domains"]
+          if len(mx_list) > 1:
+            print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain
+          mx_domain = mx_list[0]
+          self.mx_domain_to_address_domains[mx_domain].add(address_domain)
+        pass
+      else:
+        sys.stderr.write("Unknown attribute: " + `atr` + "\n")
+    # XXX is it ever permissible to have a domain with an acceptable-mx 
+    # that does not point to a TLS security policy?  If not, check/warn/fail
+    # here
+
+  def get_address_domains(self, mx_hostname):
+    labels = mx_hostname.split(".")
+    for n in range(1, len(labels)):
+      parent = "." + ".".join(labels[n:])
+      if parent in self.mx_domain_to_address_domains:
+        return self.mx_domain_to_address_domains[parent]
+    return None
+
+  def check_tls_policy_domains(self, val):
+    if type(val) != dict:
+      raise TypeError, "tls-policies should be a dict" + `val`
+    for domain, policies in val.items():
+      try:
+        assert type(domain) == unicode
+        d = str(domain) # convert from unicode
+      except:
+        raise TypeError, "tls-policy domain not a string" + `domain`
+      yield (d, policies)
+
+if __name__ == "__main__":
+  c = Config()

MTAConfigGenerator.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+
+import sys
+import string
+import os, os.path
+
+def parse_line(line_data):
+  """
+  Return the (line number, left hand side, right hand side) of a stripped
+  postfix config line.
+
+  Lines are like:
+  smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
+  """
+  num,line = line_data
+  left, sep, right = line.partition("=")
+  if not sep:
+    return None
+  return (num, left.strip(), right.strip())
+
+class MTAConfigGenerator:
+  def __init__(self, policy_config):
+    self.policy_config = policy_config
+
+class ExistingConfigError(ValueError): pass
+
+class PostfixConfigGenerator(MTAConfigGenerator):
+  def __init__(self, policy_config, postfix_dir, fixup=False):
+    self.fixup = fixup
+    self.postfix_dir = postfix_dir
+    self.postfix_cf_file = self.find_postfix_cf()
+    if not os.access(self.postfix_cf_file, os.W_OK):
+      raise Exception("Can't write to %s, please re-run as root."
+        % self.postfix_cf_file)
+    self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy")
+    self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile")
+    MTAConfigGenerator.__init__(self, policy_config)
+    self.wrangle_existing_config()
+    self.set_domainwise_tls_policies()
+    self.update_CAfile()
+    os.system("sudo service postfix reload")
+
+  def ensure_cf_var(self, var, ideal, also_acceptable):
+    """
+    Ensure that existing postfix config @var is in the list of @acceptable
+    values; if not, set it to the ideal value.
+    """
+    acceptable = [ideal] + also_acceptable
+
+    l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)]
+    if not any(l):
+      self.additions.append(var + " = " + ideal)
+    else:
+      values = map(parse_line, l)
+      if len(set(values)) > 1:
+        if self.fixup:
+          conflicting_lines = [num for num,_var,val in values]
+          self.deletions.extend(conflicting_lines)
+          self.additions.append(var + " = " + ideal)
+        else:
+          raise ExistingConfigError, "Conflicting existing config values " + `l`
+      val = values[0][2]
+      if val not in acceptable:
+        if self.fixup:
+          self.deletions.append(values[0][0])
+          self.additions.append(var + " = " + ideal)
+        else:
+          raise ExistingConfigError, "Existing config has %s=%s"%(var,val)
+
+  def wrangle_existing_config(self):
+    """
+    Try to ensure/mutate that the config file is in a sane state.
+    Fixup means we'll delete existing lines if necessary to get there.
+    """
+    self.additions = []
+    self.deletions = []
+    self.fn = self.find_postfix_cf()
+    self.raw_cf = open(self.fn).readlines()
+    self.cf = map(string.strip, self.raw_cf)
+    #self.cf = [line for line in cf if line and not line.startswith("#")]
+
+    # Check we're currently accepting inbound STARTTLS sensibly
+    self.ensure_cf_var("smtpd_use_tls", "yes", [])
+    # Ideally we use it opportunistically in the outbound direction
+    self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"])
+    # Maximum verbosity lets us collect failure information
+    self.ensure_cf_var("smtp_tls_loglevel", "1", [])
+    # Inject a reference to our per-domain policy map
+    policy_cf_entry = "texthash:" + self.policy_file
+
+    self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, [])
+    self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, [])
+
+    self.maybe_add_config_lines()
+
+  def maybe_add_config_lines(self):
+    if not self.additions:
+      return
+    if self.fixup:
+      print "Deleting lines:", self.deletions
+    self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"]
+    new_cf_lines = "\n".join(self.additions) + "\n"
+    print "Adding to %s:" % self.fn
+    print new_cf_lines
+    if self.raw_cf[-1][-1] == "\n":     sep = ""
+    else:                               sep = "\n"
+
+    self.new_cf = ""
+    for num, line in enumerate(self.raw_cf):
+      if self.fixup and num in self.deletions:
+        self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line
+      else:
+        self.new_cf += line
+    self.new_cf += sep + new_cf_lines
+
+    f = open(self.fn, "w")
+    f.write(self.new_cf)
+    f.close()
+
+  def find_postfix_cf(self):
+    "Search far and wide for the correct postfix configuration file"
+    return os.path.join(self.postfix_dir, "main.cf")
+
+  def set_domainwise_tls_policies(self):
+    self.policy_lines = []
+    for address_domain, properties in self.policy_config.acceptable_mxs.items():
+      mx_list = properties["accept-mx-domains"]
+      if len(mx_list) > 1:
+        print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain
+      mx_domain = mx_list[0]
+      mx_policy = self.policy_config.tls_policies[mx_domain]
+      entry = address_domain + " encrypt"
+      if "min-tls-version" in mx_policy:
+        if mx_policy["min-tls-version"].lower() == "tlsv1":
+          entry += " protocols=!SSLv2:!SSLv3"
+        elif mx_policy["min-tls-version"].lower() == "tlsv1.1":
+          entry += " protocols=!SSLv2:!SSLv3:!TLSv1"
+        elif mx_policy["min-tls-version"].lower() == "tlsv1.2":
+          entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1"
+        else:
+          print mx_policy["min-tls-version"]
+      self.policy_lines.append(entry)
+
+    f = open(self.policy_file, "w")
+    f.write("\n".join(self.policy_lines) + "\n")
+    f.close()
+
+  def update_CAfile(self):
+    os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " +
+      self.ca_file)
+
+if __name__ == "__main__":
+  import ConfigParser
+  if len(sys.argv) != 3:
+    print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix"
+    sys.exit(1)
+  c = ConfigParser.Config(sys.argv[1])
+  postfix_dir = sys.argv[2]
+  pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True)
+  print "Done."

letsencrypt-postfix/PostfixLogSummary.py

@@ -1,10 +1,15 @@
 #!/usr/bin/env python
+import argparse
+import collections
+import os
 import re
 import sys
-import collections
+import time
 
 import Config
 
+TIME_FORMAT = "%b %d %H:%M:%S"
+
 # TODO: There's more to be learned from postfix logs!  Here's one sample
 # observed during failures from the sender vagrant vm:
 
@@ -21,7 +26,7 @@ import Config
 #
 # Also:
 # Oct 10 19:12:13 sender postfix/smtp[1711]: 62D3F481249: to=<vagrant@valid-example-recipient.com>, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.07, delays=0.03/0.01/0.03/0, dsn=4.7.4, status=deferred (TLS is required, but was not offered by host valid-example-recipient.com[192.168.33.7])
-def get_counts(input, config):
+def get_counts(input, config, earliest_timestamp):
   seen_trusted = False
 
   counts = collections.defaultdict(lambda: collections.defaultdict(int))
@@ -31,13 +36,17 @@ def get_counts(input, config):
   # indicate a problem that should be alerted on.
   # ([^[]*) <--- any group of characters that is not "["
   # Log lines for when a message is deferred for a TLS-related reason. These
-  deferred_re = re.compile("relay=([^[]*).* status=deferred.*TLS")
+  deferred_re = re.compile("relay=([^[ ]*).* status=deferred.*TLS")
   # Log lines for when a TLS connection was successfully established. These can
   # indicate the difference between Untrusted, Trusted, and Verified certs.
   connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)")
   mx_to_domain_mapping = config.get_mx_to_domain_policy_map()
 
+  timestamp = 0
   for line in sys.stdin:
+    timestamp = time.strptime(line[0:15], TIME_FORMAT)
+    if timestamp < earliest_timestamp:
+      continue
     deferred = deferred_re.search(line)
     connected = connected_re.search(line)
     if connected:
@@ -54,11 +63,7 @@ def get_counts(input, config):
     elif deferred:
       mx_hostname = deferred.group(1).lower()
       tls_deferred[mx_hostname] += 1
-  if not seen_trusted:
-    # Postfix will only emit 'Trusted' if the certificate validates according to
-    # the set of trust roots (CA certs) configured in smtp_tls_CAfile.
-    print "Didn't see any trusted connections. Need to install some trust roots?"
-  return (counts, tls_deferred)
+  return (counts, tls_deferred, seen_trusted, timestamp)
 
 def print_summary(counts):
   for mx_hostname, validations in counts.items():
@@ -68,11 +73,32 @@ def print_summary(counts):
       print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"]
 
 if __name__ == "__main__":
-  if len(sys.argv) != 2:
-    print "Usage: %s starttls-everywhere.json" % sys.argv[0]
-    sys.exit(1)
+  arg_parser = argparse.ArgumentParser(description='Detect delivery problems'
+    ' in Postfix log files that may be caused by security policies')
+  arg_parser.add_argument('-c', action="store_true", dest="cron", default=False)
+  arg_parser.add_argument("policy_file", nargs='?',
+    default=os.path.join("examples", "starttls-everywhere.json"),
+    help="STARTTLS Everywhere policy file")
+
+  args = arg_parser.parse_args()
   config = Config.Config()
-  config.load_from_json_file(sys.argv[1])
-  (counts, tls_deferred) = get_counts(sys.stdin, config)
-  print_summary(counts)
-  print tls_deferred
+  config.load_from_json_file(args.policy_file)
+
+  last_timestamp_processed = 0
+  timestamp_file = '/tmp/starttls-everywhere-last-timestamp-processed.txt'
+  if os.path.isfile(timestamp_file):
+    last_timestamp_processed = time.strptime(open(timestamp_file).read(), TIME_FORMAT)
+  (counts, tls_deferred, seen_trusted, latest_timestamp) = get_counts(sys.stdin, config, last_timestamp_processed)
+  with open(timestamp_file, "w") as f:
+    f.write(time.strftime(TIME_FORMAT, latest_timestamp))
+
+  # If not running in cron, print an overall summary of log lines seen from known hosts.
+  if not args.cron:
+    print_summary(counts)
+    if not seen_trusted:
+      print 'No Trusted connections seen! Probably need to install a CAfile.'
+
+  if len(tls_deferred) > 0:
+    print "Some mail was deferred due to TLS problems:"
+    for (k, v) in tls_deferred.iteritems():
+      print "%s: %s" % (k, v)