diff --git a/ConfigParser.py b/ConfigParser.py new file mode 100755 index 000000000..d1c413f74 --- /dev/null +++ b/ConfigParser.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python + +import sys +import json +from datetime import datetime +import string +import collections + +def parse_timestamp(ts): + try: + int(ts) + dt = datetime.fromtimestamp(ts) + return dt + except: + raise ValueError, "Invalid timestamp integer: " + `ts` + +legal = string.letters + string.digits + ".-" +known_tlds =["com","org","net","biz","info",] # xxx make me from an ICANN list +def looks_like_a_domain(s): + "Return true if string looks like a domain, as best we can tell..." + global known_tlds + try: + domain = s.lower() + assert domain[0].islower() + assert all([c in legal for c in domain]) + tld = s.split(".")[-1] + if tld not in known_tlds: + # XXX perform DNS query to determine that this TLD exists + pass + return True + except: + return False + +class Config: + def __init__(self, cfg_file_name = "config.json"): + f = open(cfg_file_name) + self.cfg = json.loads(f.read()) + self.tls_policies = {} + self.mx_map = {} + for atr, val in self.cfg.items(): + # Verify each attribute of the structure + if atr.startswith("comment"): + continue + if atr == "author": + if type(val) not in [str, unicode]: + raise TypeError, "Author must be a string: " + `val` + elif atr == "timestamp": + self.timestamp = parse_timestamp(val) + elif atr == "expires": + self.expires = parse_timestamp(val) + elif atr == "tls-policies": + for domain, policies in self.check_tls_policy_domains(val): + if type(policies) != dict: + raise TypeError, domain + "'s policies should be a dict: " + `policies` + self.tls_policies[domain] = {} # being here enforces TLS at all + for policy, v in policies.items(): + value = str(v).lower() + if policy == "require-tls": + if value in ("true", "1", "yes"): + self.tls_policies[domain]["required"] = True + elif value in ("false", "0", "no"): + self.tls_policies[domain]["required"] = False + else: + raise ValueError, "Unknown require-tls value " + `value` + elif policy == "min-tls-version": + reasonable = ["TLS", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] + reasonable = map(string.lower, reasonable) + if not value in reasonable: + raise ValueError, "Not a valid TLS version string: " + `value` + self.tls_policies[domain]["min-tls-version"] = str(value) + elif policy == "enforce-mode": + if value == "enforce": + self.tls_policies[domain]["enforce"] = True + elif value == "log-only": + self.tls_policies[domain]["enforce"] = False + else: + raise ValueError, "Not a known enoforcement policy " + `value` + elif atr == "acceptable-mxs": + self.acceptable_mxs = val + self.mx_domain_to_address_domains = collections.defaultdict(set) + for address_domain, properties in self.acceptable_mxs.items(): + mx_list = properties["accept-mx-domains"] + if len(mx_list) > 1: + print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain + mx_domain = mx_list[0] + self.mx_domain_to_address_domains[mx_domain].add(address_domain) + pass + else: + sys.stderr.write("Unknown attribute: " + `atr` + "\n") + # XXX is it ever permissible to have a domain with an acceptable-mx + # that does not point to a TLS security policy? If not, check/warn/fail + # here + + def get_address_domains(self, mx_hostname): + labels = mx_hostname.split(".") + for n in range(1, len(labels)): + parent = "." + ".".join(labels[n:]) + if parent in self.mx_domain_to_address_domains: + return self.mx_domain_to_address_domains[parent] + return None + + def check_tls_policy_domains(self, val): + if type(val) != dict: + raise TypeError, "tls-policies should be a dict" + `val` + for domain, policies in val.items(): + try: + assert type(domain) == unicode + d = str(domain) # convert from unicode + except: + raise TypeError, "tls-policy domain not a string" + `domain` + yield (d, policies) + +if __name__ == "__main__": + c = Config() diff --git a/MTAConfigGenerator.py b/MTAConfigGenerator.py new file mode 100755 index 000000000..5ca8f5aee --- /dev/null +++ b/MTAConfigGenerator.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +import sys +import string +import os, os.path + +def parse_line(line_data): + """ + Return the (line number, left hand side, right hand side) of a stripped + postfix config line. + + Lines are like: + smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache + """ + num,line = line_data + left, sep, right = line.partition("=") + if not sep: + return None + return (num, left.strip(), right.strip()) + +class MTAConfigGenerator: + def __init__(self, policy_config): + self.policy_config = policy_config + +class ExistingConfigError(ValueError): pass + +class PostfixConfigGenerator(MTAConfigGenerator): + def __init__(self, policy_config, postfix_dir, fixup=False): + self.fixup = fixup + self.postfix_dir = postfix_dir + self.postfix_cf_file = self.find_postfix_cf() + if not os.access(self.postfix_cf_file, os.W_OK): + raise Exception("Can't write to %s, please re-run as root." + % self.postfix_cf_file) + self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy") + self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") + MTAConfigGenerator.__init__(self, policy_config) + self.wrangle_existing_config() + self.set_domainwise_tls_policies() + self.update_CAfile() + os.system("sudo service postfix reload") + + def ensure_cf_var(self, var, ideal, also_acceptable): + """ + Ensure that existing postfix config @var is in the list of @acceptable + values; if not, set it to the ideal value. + """ + acceptable = [ideal] + also_acceptable + + l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)] + if not any(l): + self.additions.append(var + " = " + ideal) + else: + values = map(parse_line, l) + if len(set(values)) > 1: + if self.fixup: + conflicting_lines = [num for num,_var,val in values] + self.deletions.extend(conflicting_lines) + self.additions.append(var + " = " + ideal) + else: + raise ExistingConfigError, "Conflicting existing config values " + `l` + val = values[0][2] + if val not in acceptable: + if self.fixup: + self.deletions.append(values[0][0]) + self.additions.append(var + " = " + ideal) + else: + raise ExistingConfigError, "Existing config has %s=%s"%(var,val) + + def wrangle_existing_config(self): + """ + Try to ensure/mutate that the config file is in a sane state. + Fixup means we'll delete existing lines if necessary to get there. + """ + self.additions = [] + self.deletions = [] + self.fn = self.find_postfix_cf() + self.raw_cf = open(self.fn).readlines() + self.cf = map(string.strip, self.raw_cf) + #self.cf = [line for line in cf if line and not line.startswith("#")] + + # Check we're currently accepting inbound STARTTLS sensibly + self.ensure_cf_var("smtpd_use_tls", "yes", []) + # Ideally we use it opportunistically in the outbound direction + self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) + # Maximum verbosity lets us collect failure information + self.ensure_cf_var("smtp_tls_loglevel", "1", []) + # Inject a reference to our per-domain policy map + policy_cf_entry = "texthash:" + self.policy_file + + self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) + self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) + + self.maybe_add_config_lines() + + def maybe_add_config_lines(self): + if not self.additions: + return + if self.fixup: + print "Deleting lines:", self.deletions + self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"] + new_cf_lines = "\n".join(self.additions) + "\n" + print "Adding to %s:" % self.fn + print new_cf_lines + if self.raw_cf[-1][-1] == "\n": sep = "" + else: sep = "\n" + + self.new_cf = "" + for num, line in enumerate(self.raw_cf): + if self.fixup and num in self.deletions: + self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line + else: + self.new_cf += line + self.new_cf += sep + new_cf_lines + + f = open(self.fn, "w") + f.write(self.new_cf) + f.close() + + def find_postfix_cf(self): + "Search far and wide for the correct postfix configuration file" + return os.path.join(self.postfix_dir, "main.cf") + + def set_domainwise_tls_policies(self): + self.policy_lines = [] + for address_domain, properties in self.policy_config.acceptable_mxs.items(): + mx_list = properties["accept-mx-domains"] + if len(mx_list) > 1: + print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain + mx_domain = mx_list[0] + mx_policy = self.policy_config.tls_policies[mx_domain] + entry = address_domain + " encrypt" + if "min-tls-version" in mx_policy: + if mx_policy["min-tls-version"].lower() == "tlsv1": + entry += " protocols=!SSLv2:!SSLv3" + elif mx_policy["min-tls-version"].lower() == "tlsv1.1": + entry += " protocols=!SSLv2:!SSLv3:!TLSv1" + elif mx_policy["min-tls-version"].lower() == "tlsv1.2": + entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" + else: + print mx_policy["min-tls-version"] + self.policy_lines.append(entry) + + f = open(self.policy_file, "w") + f.write("\n".join(self.policy_lines) + "\n") + f.close() + + def update_CAfile(self): + os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + + self.ca_file) + +if __name__ == "__main__": + import ConfigParser + if len(sys.argv) != 3: + print "Usage: MTAConfigGenerator starttls-everywhere.json /etc/postfix" + sys.exit(1) + c = ConfigParser.Config(sys.argv[1]) + postfix_dir = sys.argv[2] + pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) + print "Done." diff --git a/letsencrypt-postfix/PostfixLogSummary.py b/letsencrypt-postfix/PostfixLogSummary.py index 956a069eb..a641a9f31 100755 --- a/letsencrypt-postfix/PostfixLogSummary.py +++ b/letsencrypt-postfix/PostfixLogSummary.py @@ -1,10 +1,15 @@ #!/usr/bin/env python +import argparse +import collections +import os import re import sys -import collections +import time import Config +TIME_FORMAT = "%b %d %H:%M:%S" + # TODO: There's more to be learned from postfix logs! Here's one sample # observed during failures from the sender vagrant vm: @@ -21,7 +26,7 @@ import Config # # Also: # Oct 10 19:12:13 sender postfix/smtp[1711]: 62D3F481249: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.07, delays=0.03/0.01/0.03/0, dsn=4.7.4, status=deferred (TLS is required, but was not offered by host valid-example-recipient.com[192.168.33.7]) -def get_counts(input, config): +def get_counts(input, config, earliest_timestamp): seen_trusted = False counts = collections.defaultdict(lambda: collections.defaultdict(int)) @@ -31,13 +36,17 @@ def get_counts(input, config): # indicate a problem that should be alerted on. # ([^[]*) <--- any group of characters that is not "[" # Log lines for when a message is deferred for a TLS-related reason. These - deferred_re = re.compile("relay=([^[]*).* status=deferred.*TLS") + deferred_re = re.compile("relay=([^[ ]*).* status=deferred.*TLS") # Log lines for when a TLS connection was successfully established. These can # indicate the difference between Untrusted, Trusted, and Verified certs. connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") mx_to_domain_mapping = config.get_mx_to_domain_policy_map() + timestamp = 0 for line in sys.stdin: + timestamp = time.strptime(line[0:15], TIME_FORMAT) + if timestamp < earliest_timestamp: + continue deferred = deferred_re.search(line) connected = connected_re.search(line) if connected: @@ -54,11 +63,7 @@ def get_counts(input, config): elif deferred: mx_hostname = deferred.group(1).lower() tls_deferred[mx_hostname] += 1 - if not seen_trusted: - # Postfix will only emit 'Trusted' if the certificate validates according to - # the set of trust roots (CA certs) configured in smtp_tls_CAfile. - print "Didn't see any trusted connections. Need to install some trust roots?" - return (counts, tls_deferred) + return (counts, tls_deferred, seen_trusted, timestamp) def print_summary(counts): for mx_hostname, validations in counts.items(): @@ -68,11 +73,32 @@ def print_summary(counts): print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"] if __name__ == "__main__": - if len(sys.argv) != 2: - print "Usage: %s starttls-everywhere.json" % sys.argv[0] - sys.exit(1) + arg_parser = argparse.ArgumentParser(description='Detect delivery problems' + ' in Postfix log files that may be caused by security policies') + arg_parser.add_argument('-c', action="store_true", dest="cron", default=False) + arg_parser.add_argument("policy_file", nargs='?', + default=os.path.join("examples", "starttls-everywhere.json"), + help="STARTTLS Everywhere policy file") + + args = arg_parser.parse_args() config = Config.Config() - config.load_from_json_file(sys.argv[1]) - (counts, tls_deferred) = get_counts(sys.stdin, config) - print_summary(counts) - print tls_deferred + config.load_from_json_file(args.policy_file) + + last_timestamp_processed = 0 + timestamp_file = '/tmp/starttls-everywhere-last-timestamp-processed.txt' + if os.path.isfile(timestamp_file): + last_timestamp_processed = time.strptime(open(timestamp_file).read(), TIME_FORMAT) + (counts, tls_deferred, seen_trusted, latest_timestamp) = get_counts(sys.stdin, config, last_timestamp_processed) + with open(timestamp_file, "w") as f: + f.write(time.strftime(TIME_FORMAT, latest_timestamp)) + + # If not running in cron, print an overall summary of log lines seen from known hosts. + if not args.cron: + print_summary(counts) + if not seen_trusted: + print 'No Trusted connections seen! Probably need to install a CAfile.' + + if len(tls_deferred) > 0: + print "Some mail was deferred due to TLS problems:" + for (k, v) in tls_deferred.iteritems(): + print "%s: %s" % (k, v)