upstream/certbot
1
0
Fork 0
mirror of https://github.com/certbot/certbot.git synced 2026-06-07 07:42:08 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'github/master'

Browse source
This commit is contained in:
Peter Eckersley 2016-02-18 18:58:45 -08:00
commit 959e943de8
14 changed files with 8058 additions and 303 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

91
CheckSTARTTLS.py
View file

@ -7,9 +7,14 @@ import socket
import subprocess
import re
import json
import collections
import dns.resolver
from M2Crypto import X509
from publicsuffix import PublicSuffixList
public_suffix_list = PublicSuffixList()
CERTS_OBSERVED = 'certs-observed'
def mkdirp(path):
try:
@ -20,7 +25,7 @@ def mkdirp(path):
else: raise
def extract_names(pem):
"""Return a list of DNS subject names from PEM-encoded leaf cert."""
"""Return a set of DNS subject names from PEM-encoded leaf cert."""
leaf = X509.load_cert_string(pem, X509.FORMAT_PEM)
subj = leaf.get_subject()
@ -59,7 +64,7 @@ def tls_connect(mx_host, mail_domain):
return
# Save a copy of the certificate for later analysis
with open(os.path.join(mail_domain, mx_host), "w") as f:
with open(os.path.join(CERTS_OBSERVED, mail_domain, mx_host), "w") as f:
f.write(output)
def valid_cert(filename):
@ -71,6 +76,8 @@ def valid_cert(filename):
if open(filename).read().find("-----BEGIN CERTIFICATE-----") == -1:
return False
try:
# The file contains both the leaf cert and any intermediates, so we pass it
# as both the cert to validate and as the "untrusted" chain.
output = subprocess.check_output("""openssl verify -CApath /home/jsha/mozilla/ -purpose sslserver \
-untrusted "%s" \
"%s"
@ -80,17 +87,25 @@ def valid_cert(filename):
return False
def check_certs(mail_domain):
"""
Return "" if any certs for any mx domains pointed to by mail_domain
were invalid, and a public suffix for one if they were all valid
"""
dir = os.path.join(CERTS_OBSERVED, mail_domain)
if not os.path.exists(dir):
collect(mail_domain)
names = set()
for mx_hostname in os.listdir(mail_domain):
filename = os.path.join(mail_domain, mx_hostname)
for mx_hostname in os.listdir(dir):
filename = os.path.join(dir, mx_hostname)
if not valid_cert(filename):
return ""
else:
new_names = extract_names_from_openssl_output(filename)
new_names = set(public_suffix_list.get_public_suffix(n) for n in new_names)
names.update(new_names)
names.add(filename.rstrip("."))
if len(names) >= 1:
return common_suffix(names)
# Hack: Just pick an arbitrary suffix for now. Do something cleverer later.
return names.pop()
else:
return ""
@ -120,21 +135,32 @@ def supports_starttls(mx_host):
except socket.error as e:
print "Connection to %s failed: %s" % (mx_host, e.strerror)
return False
except smtplib.SMTPException:
print "No STARTTLS support on %s" % mx_host
except smtplib.SMTPException, e:
# In order to talk to some hosts, you need to run this from a host that has a
# reverse DNS entry. AWS instances all have reverse DNS, as an example.
if e[0] == 554:
print e[1]
else:
print "No STARTTLS support on %s" % mx_host, e[0]
return False
def min_tls_version(mail_domain):
protocols = []
for mx_hostname in os.listdir(mail_domain):
filename = os.path.join(mail_domain, mx_hostname)
for mx_hostname in os.listdir(os.path.join(CERTS_OBSERVED, mail_domain)):
filename = os.path.join(CERTS_OBSERVED, mail_domain, mx_hostname)
contents = open(filename).read()
protocol = re.findall("Protocol : (.*)", contents)[0]
protocols.append(protocol)
return min(protocols)
def collect(mail_domain):
mkdirp(mail_domain)
"""
Attempt to connect to each MX hostname for mail_doman and negotiate STARTTLS.
Store the output in a directory with the same name as mail_domain to make
subsequent analysis faster.
"""
print "Checking domain %s" % mail_domain
mkdirp(os.path.join(CERTS_OBSERVED, mail_domain))
answers = dns.resolver.query(mail_domain, 'MX')
for rdata in answers:
mx_host = str(rdata.exchange).rstrip(".")
@ -142,29 +168,24 @@ def collect(mail_domain):
if __name__ == '__main__':
"""Consume a target list of domains and output a configuration file for those domains."""
if len(sys.argv) == 1:
print("Please pass at least one mail domain as an argument")
if len(sys.argv) < 2:
print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json")
config = {
"address-domains": {
},
"mx-domains": {
}
}
for domain in sys.argv[1:]:
collect(domain)
if len(os.listdir(domain)) == 0:
continue
suffix = check_certs(domain)
min_version = min_tls_version(domain)
if suffix != "":
suffix_match = "*." + suffix
config["address-domains"][domain] = {
"accept-mx-domains": [suffix_match]
}
config["mx-domains"][suffix_match] = {
"require-tls": True,
"min-tls-version": min_version
}
config = collections.defaultdict(dict)
print json.dumps(config, indent=2)
for input in sys.argv[1:]:
for domain in open(input).readlines():
domain = domain.strip()
suffix = check_certs(domain)
if suffix != "":
min_version = min_tls_version(domain)
suffix_match = "." + suffix
config["acceptable-mxs"][domain] = {
"accept-mx-domains": [suffix_match]
}
config["tls-policies"][suffix_match] = {
"require-tls": True,
"min-tls-version": min_version
}
print json.dumps(config, indent=2, sort_keys=True)

568
Config.py Normal file
View file

@ -0,0 +1,568 @@
from datetime import datetime
from dateutil import parser as dateutil_parser
import collections
import json
import logging
import pprint
"""Idea here being to start with something that is decomposed so it's easier to
make do json in *and* out, differences between configs and config extension.
"""
#TODO scope logging and handlers better, control verbosity by command line flags
logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler())
def parse_bool_from_json(value, attr_name):
if value in ('true', '1', 1, 'yes'):
bool_value = True
elif value in ('false', '0', 0, 'no'):
bool_value = False
elif value in (True, False):
bool_value = value
else:
raise ConfigError('Config value %s is an invalid boolean value.' % attr_name)
return bool_value
def parse_timestamp(value, attr_name):
if isinstance(value, datetime):
return value
try:
ts = int(value)
return datetime.fromtimestamp(ts)
except (TypeError, ValueError):
pass
try:
return dateutil_parser.parse(value)
except (TypeError, ValueError):
raise ConfigError('Config value %s is an invalid date or timestamp.' % attr_name)
def verify_member_of(value, member_list, attr_name):
if value not in member_list:
raise ConfigError('Config value "%s" must be one of (%s)' % (
attr_name, ', '.join(member_list))
)
return value
def verify_string(value, attr_name, max_length=200):
if not isinstance(value, (str, unicode)):
raise ConfigError('Config value %s must be a string.' % attr_name)
if len(value) > max_length:
raise ConfigError('Config value %s is too long.' % attr_name)
return value
def to_dict(config_dict):
"""Cleans up BaseConfig children to be serialized."""
d = {}
for key, val in config_dict.iteritems():
if isinstance(val, BaseConfig):
d[key] = to_dict(val._data)
elif isinstance(val, datetime):
d[key] = val.strftime('%Y-%m-%dT%H:%M:%S%z')
elif isinstance(val, dict):
d[key] = to_dict(val)
else:
d[key] = val
return d
class BaseConfig(object):
"""Top level config class for common methods.
Requirements for using class:
- list all properties with getters *and* setters in class
variable 'config_properties'
- __init__ of child classes must be callable with *only*
keyword arguments to allow method calls to update to create
a new config
... more ...
"""
config_properties = []
def __init__(self):
# container for validated properties with JSON names
self._data = {}
def __repr__(self):
s = '< %s %s >' % (self.__class__.__name__,
pprint.pformat(self._data))
return s
def update(self, newer_config, merge=False, **kwargs):
"""Create a fresh config combining the new and old configs.
It does this by iterating over the 'config_properties' class
attribute which contains names of property attributes for the config.
Two methods of combining configs are possible, an 'update' and
a 'merge', the latter set by the keyword argument 'merge=True'.
An update overrides older values with new values -- even if those
new values are None. Update will remove values that are present in
the old config if they are not present in the new config.
A merge by comparison will allow old values to persist if they are
not specified in the new config. This can be used for end-user
customizations to override specific settings without having to re-create
large portions of a config to override it.
Arguments:
newer_config: A config object to combine with the current config.
merge: Allows old values not overridden to survive into the fresh config.
Returns:
A config object of the same sort as called upon.
"""
# removed 'merge' kw arg - and it was passed to constructor
# make a note to not do that, consume it on the param list
fresh_config = self.__class__(**kwargs)
logger.debug('from parent update kwargs %s' % kwargs)
logger.debug('from parent update merge %s' % merge)
if not isinstance(newer_config, self.__class__):
raise ConfigError('Attempting to update a %s with a %s' % (
self.__class__,
newer_config.__class__))
for prop_name in self.config_properties:
# get the specified property off of the current class
prop = self.__class__.__dict__.get(prop_name)
assert prop
new_value = prop.fget(newer_config)
old_value = prop.fget(self)
if new_value is not None:
prop.fset(fresh_config, new_value)
elif merge and old_value is not None:
prop.fset(fresh_config, old_value)
return fresh_config
def merge(self, newer_config, **kwargs):
"""Combines configs and keeps old values if they are not overridden.
See docstring for 'update' method for more details.
Arguments:
newer_config: A config object to combine with the current config.
merge: Allows old values not overridden to survive into the fresh config.
Returns:
A config object of the same sort as called upon.
"""
kwargs['merge'] = True
logger.debug('from parent merge: %s' % kwargs)
return self.update(newer_config, **kwargs)
def to_json(self):
d = to_dict(self._data)
return json.dumps(d)
def write_to_json_file(self, json_filename, f_open=open):
data = self.to_json()
try:
with f_open(json_filename, 'w') as f:
f.write(data)
except IOError:
raise
def load_from_json_file(self, json_filename, f_open=open):
try:
with f_open(json_filename, 'r') as f:
json_str = f.read()
json_dict = json.loads(json_str)
except IOError:
raise
except ValueError:
raise ConfigError('No valid JSON found in file: %s' % json_filename)
self.from_json_dict(json_dict)
def from_json_dict(self, json_dict):
raise NotImplmented('BaseConfig should not be populated.')
class Config(BaseConfig):
"""Config container for StartTLS Everywhere configuration.
Intended as a simple container that unifies where validatation occurs,
and is capable of comparing configs to warn of things like changing
certificate fingerprints from one scan to the next.
There is a one to one mapping of the object attributes to the JSON
object keys, albeit with dashes replaced with underscores.
"""
def __init__(self):
super(self.__class__, self).__init__()
self._data['tls-policies'] = {}
self._data['acceptable-mxs'] = {}
def __add__(self, other_config):
"""Allow addition but not really of *full* configs, need to flesh that out."""
#TODO add this
raise NotImplemented
def update(self, other_config):
"""Update properties of config from a 'newer' config and force verification."""
#TODO add this
new_config = Config()
raise NotImplemented
def from_json_dict(self, json_dict):
"""Assign JSON data to Config properties and declare sub-objects.
Let's property verification methods do the heavy lifting and mostly
maps between the JSON config names and attributes. Keeps track of
unused variables and warns about them.
"""
for key, val in json_dict.iteritems():
if key == 'author':
self.author = val
elif key == 'comment':
self.comment = val
elif key == 'expires':
self.expires = val
elif key == 'timestamp':
self.timestamp = val
elif key == 'tls-policies':
self.make_tls_policy_dict(val)
elif key == 'acceptable-mxs':
self.make_acceptable_mxs_dict(val)
else:
logger.warn('Unknown attribute "%s", skipping' % key)
@property
def author(self):
return self._data.get('author')
@author.setter
def author(self, value):
self._data['author'] = verify_string(value, 'author')
@property
def comment(self):
return self._data.get('comment')
@comment.setter
def comment(self, value):
self._data['comment'] = verify_string(value, 'comment')
@property
def expires(self):
return self._data.get('expires')
@expires.setter
def expires(self, value):
self._data['expires'] = parse_timestamp(value, 'expires')
@property
def timestamp(self):
return self._data.get('timestamp')
@timestamp.setter
def timestamp(self, value):
self._data['timestamp'] = parse_timestamp(value, 'timestamp')
@property
def tls_policies(self):
return self._data.get('tls-policies')
@property
def acceptable_mxs(self):
return self._data.get('acceptable-mxs')
def make_tls_policy_dict(self, policy_dict):
tls_policy_dict = self.tls_policies
for domain_suffix, settings in policy_dict.iteritems():
new_domain_policy = TLSPolicy(domain_suffix)
try:
new_domain_policy.from_json_dict(settings)
except ConfigError as e:
raise
tls_policy_dict[domain_suffix] = new_domain_policy
def get_tls_policy(self, mx_domain):
return self.tls_policies.get(mx_domain)
def make_acceptable_mxs_dict(self, mxs_dict):
acceptable_mxs_dict = self._data['acceptable-mxs']
for domain, settings in mxs_dict.iteritems():
new_domain_policy = AcceptableMX(domain)
try:
new_domain_policy.from_json_dict(settings)
except ConfigError as e:
raise
acceptable_mxs_dict[domain] = new_domain_policy
def get_address_domains(self, mx_hostname, mx_to_domain_map):
"""Do a fuzzy DNS host match on provided map to get lists of policies.
Args:
mx_hostname (string): The hostname from an MX record.
mx_to_domain_map: Mapping from MX hosts to AcceptableMX
policies, the same AcceptableMX policy may occur more
than once. e.g. {'mx_host3': set(AcceptableMX, ...)}
The map can be generated by Config.get_mx_to_domain_policy_map.
Returns:
The set containing all AcceptableMX policies that list the
provided MX host as viable.
"""
labels = mx_hostname.split(".")
for n in range(1, len(labels)):
parent = "." + ".".join(labels[n:])
if parent in mx_to_domain_map:
return mx_to_domain_map[parent]
return None
def get_mx_to_domain_policy_map(self):
"""Create mapping of MX hostnames to sets of AcceptableMX policies.
Generate a dictionary that is typically used in log analysis
(e.g. if your MTA logs interact with beta.innotech.com you use
this mapping to tell you it used the innotech.com AcceptableMX
policy or policies). There are of course complications.
"""
# create reverse mapping dictionary as well for auditing
# and reviewing logs
mx_to_domain_policy = collections.defaultdict(set)
for mx_host, domain_policy in self.get_all_mx_items():
existing_mx_policies = mx_to_domain_policy.get(mx_host)
if existing_mx_policies:
existing_domains = [ e.domain for e in existing_mx_policies ]
if domain_policy.domain not in existing_domains:
#TODO plenty of room to enforce a security policy here
# this is also the case of google apps personal domains
msg = ('Attempting to add domain policy (%s) for MX host but MX'
' host already has a domain policy (%s), appending...')
logger.debug(msg % (domain_policy.domain,
', '.join(existing_domains)))
mx_to_domain_policy[mx_host].add(domain_policy)
return mx_to_domain_policy
def get_all_mx_items(self):
"""Iterate over (mx_host, mx_policy) - be sure to dedup!"""
all_mx_items = []
for policy in self.acceptable_mxs.values():
accepted_mxs = policy.accept_mx_domains
all_mx_items.extend([(mx_host, policy)
for mx_host in accepted_mxs])
return all_mx_items
def get_all_mx_hosts(self):
all_mx_hosts = []
[ all_mx_hosts.extend(domain_policy.acceptable_mxs)
for domain_policy in self.acceptable_mxs.values() ]
return all_mx_hosts
def is_valid(self):
#TODO implement checks to make sure domains don't overlap
#TODO add debug logging for troubleshooting sake
for mx_config in self.acceptable_mxs.values():
if not mx_config.is_valid():
return False
for domain_suffix in mx_config.accept_mx_domains:
# check to make sure every accepted MX has a TLS policy
if not domain_suffix in self.tls_policies:
return False
all_mx_hosts = self.get_all_mx_hosts()
for domain_suffix, tls_config in self.tls_policies.iteritems():
if not tls_config.is_valid():
return False
# make sure no unclaimed TLS policies have made their way in
if domain_suffix not in all_mx_hosts:
return False
return True
class TLSPolicy(BaseConfig):
ENFORCE_MODES = ('enforce', 'log-only')
TLS_VERSIONS = ('TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3')
config_properties = ['comment', 'enforce_mode', 'min_tls_version',
'require_tls', 'require_valid_certificate']
def __init__(self, domain_suffix=None):
super(self.__class__, self).__init__()
self.domain_suffix = domain_suffix
#TODO add support for two designed but yet unsupported attrs
# self._data['accept-spki-hashs'] = None
# self._data['error-notification'] = None
def from_json_dict(self, json_dict):
for key, val in json_dict.iteritems():
if key == 'comment':
self.comment = val
elif key == 'enforce-mode':
self.enforce_mode = val
elif key == 'min-tls-version':
self.min_tls_version = val
elif key == 'require-tls':
self.require_tls = val
elif key == 'require-valid-certificate':
self.require_valid_certificate = val
else:
logger.warn('Unknown key %s' % key)
def is_valid(self):
"""Do simple check that config contains all required values.
Should find a way to expose easily which config values
are required, at least place in error messages such that
incomplete configs will expose it.
"""
required_attrs = ('enforce-mode', 'min-tls-version',
'require-tls')
values_set = [self._data.get(attr) for attr in required_attrs]
if not all(values_set):
return False
else:
return True
def update(self, newer_policy, **kwargs):
if not kwargs.get('domain_suffix'):
kwargs['domain_suffix'] = self.domain_suffix
fresh_policy = super(self.__class__, self).update(newer_policy,
**kwargs)
logger.debug('from TLS child update %s' % kwargs)
return fresh_policy
def merge(self, newer_policy, **kwargs):
logger.debug('from TLS child merge: %s' % kwargs)
fresh_policy = super(self.__class__, self).merge(newer_policy,
domain_suffix=self.domain_suffix)
return fresh_policy
@property
def comment(self):
return self._data.get('comment')
@comment.setter
def comment(self, value):
self._data['comment'] = verify_string(value, 'comment')
@property
def enforce_mode(self):
return self._data.get('enforce-mode')
@enforce_mode.setter
def enforce_mode(self, value):
self._data['enforce-mode'] = verify_member_of(value, self.ENFORCE_MODES, 'enforce-mode')
@property
def min_tls_version(self):
return self._data.get('min-tls-version')
@min_tls_version.setter
def min_tls_version(self, value):
"""TODO: Should this be dealing only with strings processed by map ... lower()?"""
tls_versions = [ver.lower() for ver in self.TLS_VERSIONS]
tls_versions.extend(self.TLS_VERSIONS)
self._data['min-tls-version'] = verify_member_of(value, tls_versions, 'min-tls-version')
@property
def require_tls(self):
return self._data.get('require-tls')
@require_tls.setter
def require_tls(self, value):
self._data['require-tls'] = parse_bool_from_json(value, 'require-tls')
@property
def require_valid_certificate(self):
return self._data.get('require-valid-certificate')
@require_valid_certificate.setter
def require_valid_certificate(self, value):
self._data['require-valid-certificate'] = parse_bool_from_json(value, 'require-valid-certificate')
class AcceptableMX(BaseConfig):
"""Holds acceptable MX domain suffixes for a single mail serving domain.
Such as for gmail.com that single mail serving suffix domain is:
gmail-smtp-in.l.google.com.
Configuration of the acceptable MX suffix domains must match up with TLS policies
for the suffix domains.
"""
def __init__(self, domain=None):
super(self.__class__, self).__init__()
self.domain = domain
self._data['accept-mx-domains'] = []
@property
def accept_mx_domains(self):
return self._data.get('accept-mx-domains')
def add_acceptable_mx(self, domain_suffix):
unique_domain_suffixes = set(self._data['accept-mx-domains'])
unique_domain_suffixes.add(domain_suffix)
self._data['accept-mx-domains'] = list(unique_domain_suffixes)
@property
def comment(self):
return self._data.get('comment')
@comment.setter
def comment(self, value):
self._data['comment'] = verify_string(value, 'comment')
def is_valid(self):
"""Check to make sure there is one acceptable domain suffix.
This will need to be updated once we can actually test and support
for more than one acceptable domain suffix.
TODO: could make this object double check the data it is given with
DNS queries.
"""
if len(self._data['accept-mx-domains']) != 1:
return False
else:
return True
def from_json_dict(self, json_dict):
for key, val in json_dict.iteritems():
if key == 'accept-mx-domains':
if isinstance(val, list):
for domain_suffix in val:
self.add_acceptable_mx(domain_suffix)
else:
self.add_acceptable_mx(val)
elif key == 'comment':
self.comment = val
else:
logger.warn('warning: unknown key %s' % key)
def update(self, newer_policy, **kwargs):
logger.debug('from MX child update got %s' % kwargs)
if not kwargs.get('domain'):
kwargs['domain'] = self.domain
fresh_policy = super(self.__class__, self).update(newer_policy,
**kwargs)
if kwargs.get('merge'):
new_accepted_mxs = set(self.accept_mx_domains)
new_accepted_mxs = new_accepted_mxs.union(newer_policy.accept_mx_domains)
else:
new_accepted_mxs = newer_policy.accept_mx_domains
for domain in new_accepted_mxs:
fresh_policy.add_acceptable_mx(domain)
return fresh_policy
def merge(self, newer_policy, **kwargs):
logger.debug('from MX child merge: %s' % kwargs)
fresh_policy = super(self.__class__, self).merge(newer_policy,
**kwargs)
return fresh_policy
class ConfigError(ValueError):
def __init__(self, message):
super(self.__class__, self).__init__(message)

10
ConfigParser.py
View file

@ -93,11 +93,11 @@ class Config:
print self.tls_policies
def get_address_domains(self, mx_hostname):
for mx_domain, address_domains in self.mx_domain_to_address_domains.items():
# TODO: write this better
if (mx_hostname.find(mx_domain) > 0 and
mx_hostname.find(mx_domain) == len(mx_hostname) - len(mx_domain)):
return address_domains
labels = mx_hostname.split(".")
for n in range(1, len(labels)):
parent = "." + ".".join(labels[n:])
if parent in self.mx_domain_to_address_domains:
return self.mx_domain_to_address_domains[parent]
return None
def check_tls_policy_domains(self, val):

391
MTAConfigGenerator.py
View file

@ -1,138 +1,295 @@
#!/usr/bin/env python
import sys
import string
import os, os.path
POSTFIX_DIR = "postfix-copy"
DEFAULT_POLICY_FILE = os.path.join(POSTFIX_DIR, "starttls_everywhere_policy")
POLICY_CF_ENTRY="texthash:" + DEFAULT_POLICY_FILE
def parse_line(line_data):
"""
return the (line number, left hand side, right hand side)
of a stripped postfix config line"""
# lines are like:
# smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
num,line = line_data
left, sep, right = line.partition("=")
if not sep:
return None
return (num, left.strip(), right.strip())
"""
Return the (line number, left hand side, right hand side) of a stripped
postfix config line.
Lines are like:
smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
"""
num,line = line_data
left, sep, right = line.partition("=")
if not sep:
return None
return (num, left.strip(), right.strip())
class MTAConfigGenerator:
def __init__(self, policy_config):
self.policy_config = policy_config
def __init__(self, policy_config):
self.policy_config = policy_config
class ExistingConfigError(ValueError): pass
class PostfixConfigGenerator(MTAConfigGenerator):
def __init__(self, policy_config, fixup=False):
self.fixup = fixup
MTAConfigGenerator.__init__(self, policy_config)
self.postfix_cf_file = self.find_postfix_cf()
self.wrangle_existing_config()
self.set_domainwise_tls_policies()
os.system("sudo service postfix reload")
def __init__(self, policy_config, postfix_dir, fixup=False):
self.fixup = fixup
self.postfix_dir = postfix_dir
self.policy_file = os.path.join(postfix_dir, "starttls_everywhere_policy")
MTAConfigGenerator.__init__(self, policy_config)
def ensure_cf_var(self, var, ideal, also_acceptable):
"""
Ensure that existing postfix config @var is in the list of @acceptable
values; if not, set it to the ideal value.
"""
acceptable = [ideal] + also_acceptable
def ensure_cf_var(self, var, ideal, also_acceptable):
"""
Ensure that existing postfix config @var is in the list of @acceptable
values; if not, set it to the ideal value.
"""
acceptable = [ideal] + also_acceptable
l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)]
if not any(l):
self.additions.append(var + " = " + ideal)
else:
values = map(parse_line, l)
if len(set(values)) > 1:
if self.fixup:
#print "Scheduling deletions:" + `values`
conflicting_lines = [num for num,_var,val in values]
self.deletions.extend(conflicting_lines)
self.additions.append(var + " = " + ideal)
l = [(num,line) for num,line in enumerate(self.cf) if line.startswith(var)]
if not any(l):
self.additions.append(var + " = " + ideal)
else:
raise ExistingConfigError, "Conflicting existing config values " + `l`
val = values[0][2]
if val not in acceptable:
#print "Scheduling deletions:" + `values`
values = map(parse_line, l)
if len(set(values)) > 1:
if self.fixup:
#print "Scheduling deletions:" + `values`
conflicting_lines = [num for num,_var,val in values]
self.deletions.extend(conflicting_lines)
self.additions.append(var + " = " + ideal)
else:
raise ExistingConfigError, "Conflicting existing config values " + `l`
val = values[0][2]
if val not in acceptable:
#print "Scheduling deletions:" + `values`
if self.fixup:
self.deletions.append(values[0][0])
self.additions.append(var + " = " + ideal)
else:
raise ExistingConfigError, "Existing config has %s=%s"%(var,val)
def wrangle_existing_config(self):
"""
Try to ensure/mutate that the config file is in a sane state.
Fixup means we'll delete existing lines if necessary to get there.
"""
self.additions = []
self.deletions = []
self.fn = self.find_postfix_cf()
self.raw_cf = open(self.fn).readlines()
self.cf = map(string.strip, self.raw_cf)
#self.cf = [line for line in cf if line and not line.startswith("#")]
# Check we're currently accepting inbound STARTTLS sensibly
self.ensure_cf_var("smtpd_use_tls", "yes", [])
# Ideally we use it opportunistically in the outbound direction
self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"])
# Maximum verbosity lets us collect failure information
self.ensure_cf_var("smtp_tls_loglevel", "1", [])
# Inject a reference to our per-domain policy map
policy_cf_entry = "texthash:" + self.policy_file
self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, [])
def maybe_add_config_lines(self):
if not self.additions:
return
if self.fixup:
self.deletions.append(values[0][0])
self.additions.append(var + " = " + ideal)
print "Deleting lines:", self.deletions
self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"]
new_cf_lines = "\n".join(self.additions) + "\n"
print "Adding to %s:" % self.fn
print new_cf_lines
if self.raw_cf[-1][-1] == "\n": sep = ""
else: sep = "\n"
self.new_cf = ""
for num, line in enumerate(self.raw_cf):
if self.fixup and num in self.deletions:
self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line
else:
self.new_cf += line
self.new_cf += sep + new_cf_lines
#print self.new_cf
f = open(self.fn, "w")
f.write(self.new_cf)
f.close()
def find_postfix_cf(self):
"Search far and wide for the correct postfix configuration file"
return os.path.join(self.postfix_dir, "main.cf")
def set_domainwise_tls_policies(self):
self.policy_lines = []
all_acceptable_mxs = self.policy_config.acceptable_mxs
for address_domain, properties in all_acceptable_mxs.items():
mx_list = properties.accept_mx_domains
if len(mx_list) > 1:
print "Lists of multiple accept-mx-domains not yet supported."
print "Using MX %s for %s" % (mx_list[0], address_domain)
print "Ignoring: %s" % (', '.join(mx_list[1:]))
mx_domain = mx_list[0]
mx_policy = self.policy_config.get_tls_policy(mx_domain)
entry = address_domain + " encrypt"
if mx_policy.min_tls_version.lower() == "tlsv1":
entry += " protocols=!SSLv2,!SSLv3"
elif mx_policy.min_tls_version.lower() == "tlsv1.1":
entry += " protocols=!SSLv2,!SSLv3,!TLSv1"
elif mx_policy.min_tls_version.lower() == "tlsv1.2":
entry += " protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1"
else:
print mx_policy.min_tls_version
self.policy_lines.append(entry)
f = open(self.policy_file, "w")
f.write("\n".join(self.policy_lines) + "\n")
f.close()
### Let's Encrypt client IPlugin ###
def prepare(self):
"""Prepare the plugin.
Finish up any additional initialization.
:raises .PluginError:
when full initialization cannot be completed.
:raises .MisconfigurationError:
when full initialization cannot be completed. Plugin will
be displayed on a list of available plugins.
:raises .NoInstallationError:
when the necessary programs/files cannot be located. Plugin
will NOT be displayed on a list of available plugins.
:raises .NotSupportedError:
when the installation is recognized, but the version is not
currently supported.
"""
# XXX ensure we raise the right kinds of exceptions
self.postfix_cf_file = self.find_postfix_cf()
def more_info(self):
"""Human-readable string to help the user.
Should describe the steps taken and any relevant info to help the user
decide which plugin to use.
:rtype str:
"""
### Let's Encrypt client IInstaller ###
def get_all_names(self):
"""Returns all names that may be authenticated.
:rtype: `list` of `str`
"""
def deploy_cert(self, domain, _cert_path, key_path, _chain_path, fullchain_path):
"""Deploy certificate.
:param str domain: domain to deploy certificate file
:param str cert_path: absolute path to the certificate file
:param str key_path: absolute path to the private key file
:param str chain_path: absolute path to the certificate chain file
:param str fullchain_path: absolute path to the certificate fullchain
file (cert plus chain)
:raises .PluginError: when cert cannot be deployed
"""
self.wrangle_existing_config()
self.ensure_cf_var("smtpd_tls_cert_file", fullchain_path, [])
self.ensure_cf_var("smtpd_tls_key_file", key_path, [])
self.set_domainwise_tls_policies()
def enhance(self, domain, enhancement, options=None):
"""Perform a configuration enhancement.
:param str domain: domain for which to provide enhancement
:param str enhancement: An enhancement as defined in
:const:`~letsencrypt.constants.ENHANCEMENTS`
:param options: Flexible options parameter for enhancement.
Check documentation of
:const:`~letsencrypt.constants.ENHANCEMENTS`
for expected options for each enhancement.
:raises .PluginError: If Enhancement is not supported, or if
an error occurs during the enhancement.
"""
def supported_enhancements(self):
"""Returns a list of supported enhancements.
:returns: supported enhancements which should be a subset of
:const:`~letsencrypt.constants.ENHANCEMENTS`
:rtype: :class:`list` of :class:`str`
"""
def get_all_certs_keys(self):
"""Retrieve all certs and keys set in configuration.
:returns: tuples with form `[(cert, key, path)]`, where:
- `cert` - str path to certificate file
- `key` - str path to associated key file
- `path` - file path to configuration file
:rtype: list
"""
def save(self, title=None, temporary=False):
"""Saves all changes to the configuration files.
Both title and temporary are needed because a save may be
intended to be permanent, but the save is not ready to be a full
checkpoint. If an exception is raised, it is assumed a new
checkpoint was not created.
:param str title: The title of the save. If a title is given, the
configuration will be saved as a new checkpoint and put in a
timestamped directory. `title` has no effect if temporary is true.
:param bool temporary: Indicates whether the changes made will
be quickly reversed in the future (challenges)
:raises .PluginError: when save is unsuccessful
"""
self.maybe_add_config_lines()
def rollback_checkpoints(self, rollback=1):
"""Revert `rollback` number of configuration checkpoints.
:raises .PluginError: when configuration cannot be fully reverted
"""
def recovery_routine(self):
"""Revert configuration to most recent finalized checkpoint.
Remove all changes (temporary and permanent) that have not been
finalized. This is useful to protect against crashes and other
execution interruptions.
:raises .errors.PluginError: If unable to recover the configuration
"""
def view_config_changes(self):
"""Display all of the LE config changes.
:raises .PluginError: when config changes cannot be parsed
"""
def config_test(self):
"""Make sure the configuration is valid.
:raises .MisconfigurationError: when the config is not in a usable state
"""
def restart(self):
"""Restart or refresh the server content.
:raises .PluginError: when server cannot be restarted
"""
if os.geteuid() != 0:
os.system("sudo service postfix reload")
else:
raise ExistingConfigError, "Existing config has %s=%s"%(var,val)
def wrangle_existing_config(self):
"""
Try to ensure/mutate that the config file is in a sane state.
Fixup means we'll delete existing lines if necessary to get there.
"""
self.additions = []
self.deletions = []
self.fn = self.find_postfix_cf()
self.raw_cf = open(self.fn).readlines()
self.cf = map(string.strip, self.raw_cf)
#self.cf = [line for line in cf if line and not line.startswith("#")]
os.system("service postfix reload")
# Check we're currently accepting inbound STARTTLS sensibly
self.ensure_cf_var("smtpd_use_tls", "yes", [])
# Ideally we use it opportunistically in the outbound direction
self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"])
# Maximum verbosity lets us collect failure information
self.ensure_cf_var("smtp_tls_loglevel", "1", [])
# Inject a reference to our per-domain policy map
self.ensure_cf_var("smtp_tls_policy_maps", POLICY_CF_ENTRY, [])
self.maybe_add_config_lines()
def maybe_add_config_lines(self):
if not self.additions:
return
if self.fixup:
print "Deleting lines:", self.deletions
self.additions[:0]=["#","# New config lines added by STARTTLS Everywhere","#"]
new_cf_lines = "\n".join(self.additions) + "\n"
print "Adding to %s:" % self.fn
print new_cf_lines
if self.raw_cf[-1][-1] == "\n": sep = ""
else: sep = "\n"
self.new_cf = ""
for num, line in enumerate(self.raw_cf):
if self.fixup and num in self.deletions:
self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line
else:
self.new_cf += line
self.new_cf += sep + new_cf_lines
#print self.new_cf
f = open(self.fn, "w")
f.write(self.new_cf)
f.close()
def find_postfix_cf(self):
"Search far and wide for the correct postfix configuration file"
return os.path.join(POSTFIX_DIR,"main.cf")
def set_domainwise_tls_policies(self):
self.policy_lines = []
for address_domain, properties in self.policy_config.acceptable_mxs.items():
mx_list = properties["accept-mx-domains"]
if len(mx_list) > 1:
print "Lists of multiple accept-mx-domains not yet supported, skipping ", address_domain
mx_domain = mx_list[0]
mx_policy = self.policy_config.tls_policies[mx_domain]
entry = address_domain + " encrypt"
if "min-tls-version" in mx_policy:
entry += " " + mx_policy["min-tls-version"]
self.policy_lines.append(entry)
f = open(DEFAULT_POLICY_FILE, "w")
f.write("\n".join(self.policy_lines) + "\n")
f.close()
def usage():
print ("Usage: %s starttls-everywhere.json /etc/postfix /etc/letsencrypt/live/example.com/" %
sys.argv[0])
sys.exit(1)
if __name__ == "__main__":
import ConfigParser
c = ConfigParser.Config("starttls-everywhere.json")
pcgen = PostfixConfigGenerator(c, fixup=True)
import Config as config
if len(sys.argv) != 4:
usage()
c = config.Config()
c.load_from_json_file(sys.argv[1])
postfix_dir = sys.argv[2]
le_lineage = sys.argv[3]
pieces = [os.path.join(le_lineage, f) for f in (
"cert.pem", "privkey.pem", "chain.pem", "fullchain.pem")]
if not os.path.isdir(le_lineage) or not all(os.path.isfile(p) for p in pieces) :
print "Let's Encrypt directory", le_lineage, "does not appear to contain a valid lineage"
print
usage()
cert, key, chain, fullchain = pieces
pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True)
pcgen.prepare()
pcgen.deploy_cert("example.com", cert, key, chain, fullchain)
pcgen.save()
pcgen.restart()

73
PostfixLogSummary.py
View file

@ -3,31 +3,76 @@ import re
import sys
import collections
import ConfigParser
import Config
# TODO: There's more to be learned from postfix logs! Here's one sample
# observed during failures from the sender vagrant vm:
# Jun 6 00:21:31 precise32 postfix/smtpd[3648]: connect from localhost[127.0.0.1]
# Jun 6 00:21:34 precise32 postfix/smtpd[3648]: lost connection after STARTTLS from localhost[127.0.0.1]
# Jun 6 00:21:34 precise32 postfix/smtpd[3648]: disconnect from localhost[127.0.0.1]
# Jun 6 00:21:56 precise32 postfix/master[3001]: reload -- version 2.9.6, configuration /etc/postfix
# Jun 6 00:22:01 precise32 postfix/pickup[3674]: AF3B6480475: uid=0 from=<root>
# Jun 6 00:22:01 precise32 postfix/cleanup[3680]: AF3B6480475: message-id=<20140606002201.AF3B6480475@sender.example.com>
# Jun 6 00:22:01 precise32 postfix/qmgr[3673]: AF3B6480475: from=<root@sender.example.com>, size=576, nrcpt=1 (queue active)
# Jun 6 00:22:01 precise32 postfix/smtp[3682]: SSL_connect error to valid-example-recipient.com[192.168.33.7]:25: -1
# Jun 6 00:22:01 precise32 postfix/smtp[3682]: warning: TLS library problem: 3682:error:140740BF:SSL routines:SSL23_CLIENT_HELLO:no protocols available:s23_clnt.c:381:
# Jun 6 00:22:01 precise32 postfix/smtp[3682]: AF3B6480475: to=<vagrant@valid-example-recipient.com>, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.06, delays=0.03/0.03/0/0, dsn=4.7.5, status=deferred (Cannot start TLS: handshake failure)
#
# Also:
# Oct 10 19:12:13 sender postfix/smtp[1711]: 62D3F481249: to=<vagrant@valid-example-recipient.com>, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.07, delays=0.03/0.01/0.03/0, dsn=4.7.4, status=deferred (TLS is required, but was not offered by host valid-example-recipient.com[192.168.33.7])
def get_counts(input, config):
seen_trusted = False
counts = collections.defaultdict(lambda: collections.defaultdict(int))
r = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)")
tls_deferred = collections.defaultdict(int)
# Typical line looks like:
# Jun 12 06:24:14 sender postfix/smtp[9045]: Untrusted TLS connection established to valid-example-recipient.com[192.168.33.7]:25: TLSv1.1 with cipher AECDH-AES256-SHA (256/256 bits)
# indicate a problem that should be alerted on.
# ([^[]*) <--- any group of characters that is not "["
# Log lines for when a message is deferred for a TLS-related reason. These
deferred_re = re.compile("relay=([^[]*).* status=deferred.*TLS")
# Log lines for when a TLS connection was successfully established. These can
# indicate the difference between Untrusted, Trusted, and Verified certs.
connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)")
mx_to_domain_mapping = config.get_mx_to_domain_policy_map()
for line in sys.stdin:
result = r.search(line)
if result:
validation = result.group(1)
mx_hostname = result.group(2)
address_domains = config.get_address_domains(mx_hostname)
deferred = deferred_re.search(line)
connected = connected_re.search(line)
if connected:
validation = connected.group(1)
mx_hostname = connected.group(2).lower()
if validation == "Trusted" or validation == "Verified":
seen_trusted = True
address_domains = config.get_address_domains(mx_hostname, mx_to_domain_mapping)
if address_domains:
for d in address_domains:
counts[d][validation] += 1
counts[d]["all"] += 1
return counts
domains_str = [ a.domain for a in address_domains ]
d = ', '.join(domains_str)
counts[d][validation] += 1
counts[d]["all"] += 1
elif deferred:
mx_hostname = deferred.group(1).lower()
tls_deferred[mx_hostname] += 1
if not seen_trusted:
# Postfix will only emit 'Trusted' if the certificate validates according to
# the set of trust roots (CA certs) configured in smtp_tls_CAfile.
print "Didn't see any trusted connections. Need to install some trust roots?"
return (counts, tls_deferred)
def print_summary(counts):
for mx_hostname, validations in counts.items():
for validation, validation_count in validations.items():
if validation == "all":
continue
print mx_hostname, validation, validation_count / validations["all"]
print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"]
if __name__ == "__main__":
config = ConfigParser.Config("starttls-everywhere.json")
counts = get_counts(sys.stdin, config)
if len(sys.argv) != 2:
print "Usage: %s starttls-everywhere.json" % sys.argv[0]
sys.exit(1)
config = Config.Config()
config.load_from_json_file(sys.argv[1])
(counts, tls_deferred) = get_counts(sys.stdin, config)
print_summary(counts)
print tls_deferred

11
ProcessGoogleSTARTTLSDomains.py
View file

@ -15,8 +15,15 @@ from collections import defaultdict
csvreader = csv.reader(codecs.open(sys.argv[1], "rU", "utf-8"), delimiter=',', quotechar='"')
d = defaultdict(set)
for (address_suffix, hostname_suffix, direction, region, fraction_encrypted) in csvreader:
# Google's report doesn't include gmail.com because it's local delivery, but we
# know they support STARTTLS, so manually include them.
d["gmail.com"] = set([1])
for (address_suffix, hostname_suffix, direction, region, region_name, fraction_encrypted) in csvreader:
if direction == "outbound":
# Some domains exist in many TLDs and are summarized as, e.g. yahoo.{...}.
# We're tryingto get a solid list of the relevant TLDs, but in the meantime
# just use .com.
address_suffix = address_suffix.replace("{...}", "com")
try:
d[address_suffix].add(float(fraction_encrypted))
except ValueError:
@ -24,4 +31,4 @@ for (address_suffix, hostname_suffix, direction, region, fraction_encrypted) in
for address_suffix, fraction_encrypted in d.iteritems():
if min(fraction_encrypted) >= 0.99:
print min(fraction_encrypted), address_suffix
print address_suffix

12
README.md
View file

@ -7,6 +7,10 @@ yet!
Jacob Hoffman-Andrews <jsha@eff.org>, Peter Eckersley <pde@eff.org>, Daniel Wilcox <dmwilcox@gmail.com>
## Mailing List
starttls-everywhere@eff.org, https://lists.eff.org/mailman/listinfo/starttls-everywhere
## Background
Most email transferred between SMTP servers (aka MTAs) is transmitted in the clear and trivially interceptable. Encryption of SMTP traffic is possible using the STARTTLS mechanism, which encrypts traffic but is vulnerable to a trivial downgrade attack.
@ -32,6 +36,14 @@ STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently d
* Develop a fully-decentralized solution.
* Initially we are not engineering to scale to all mail domains on the Internet, though we believe this design can be scaled as required if large numbers of domains publish policies to it.
## Motivating examples
* [Unnammed mobile broadband provider overwrites STARTTLS flag and commands to
prevent negotiating an encrypted connection]
(https://www.techdirt.com/articles/20141012/06344928801/revealed-isps-already-violating-net-neutrality-to-block-encryption-make-everyone-less-safe-online.shtml)
* [Unknown party removes STARTTLS flag from all SMTP connections leaving
Thailand](http://www.telecomasia.net/content/google-yahoo-smtp-email-severs-hit-thailand)
## Threat model
Attacker has control of routers on the path between two MTAs of interest. Attacker cannot or will not issue valid certificates for arbitrary names. Attacker cannot or will not attack endpoints. We are trying to protect confidentiality and integrity of email transmitted over SMTP between MTAs.

131
TestConfig.py Normal file
View file

@ -0,0 +1,131 @@
import copy
import itertools
import logging
import unittest
import Config
logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler())
class TestTLSPolicy(unittest.TestCase):
def setUp(self):
self.old_config = Config.TLSPolicy(domain_suffix='.eff.org')
self.old_config.comment = 'Testing EFF.org TLS policy'
self.old_config.require_tls = True
self.old_config.require_valid_certificate = False
self.old_config.min_tls_version = 'TLSv1'
self.old_config.enforce_mode = 'log-only'
self.new_config = Config.TLSPolicy(domain_suffix='.eff.org')
self.new_config.require_valid_certificate = True
self.new_config.min_tls_version = 'TLSv1.2'
self.new_config.enforce_mode = 'enforce'
def testUpdateDropsOldSettings(self):
logger.debug('old: %s' % self.old_config)
logger.debug('new: %s' % self.new_config)
tls_policy = self.old_config.update(self.new_config)
logger.debug('just generated: %s' % tls_policy)
self.assertFalse(any([tls_policy.require_tls, tls_policy.comment]))
def testMergeKeepsOldSettings(self):
logger.debug('old: %s' % self.old_config)
logger.debug('new: %s' % self.new_config)
tls_policy = self.old_config.merge(self.new_config, merge=True)
logger.debug('just generated: %s' % tls_policy)
self.assertTrue(all([tls_policy.require_tls, tls_policy.comment]))
def testUpdateGetsNameSet(self):
tls_policy = self.old_config.update(self.new_config)
self.assertEquals(tls_policy.domain_suffix, self.old_config.domain_suffix)
class TestAcceptableMX(unittest.TestCase):
def setUp(self):
self.old_config = Config.AcceptableMX(domain='eff.org')
self.old_config.add_acceptable_mx('.eff.org')
def testUpdateDropsOldMXs(self):
new_bogus_mx = '.testing.eff.org'
new_config = Config.AcceptableMX(domain='eff.org')
new_config.add_acceptable_mx(new_bogus_mx)
updated_config = self.old_config.update(new_config)
self.assertNotIn('.eff.org', updated_config.accept_mx_domains)
def testMergeKeepsOldMXs(self):
new_bogus_mx = '.testing.eff.org'
new_config = Config.AcceptableMX(domain='eff.org')
new_config.add_acceptable_mx(new_bogus_mx)
updated_config = self.old_config.merge(new_config)
self.assertListEqual(sorted(['.eff.org', '.testing.eff.org']),
sorted(updated_config.accept_mx_domains))
def testUpdateGetsNameSet(self):
new_policy = Config.AcceptableMX(domain=self.old_config.domain)
mx_policy = self.old_config.update(new_policy)
self.assertEquals(mx_policy.domain, self.old_config.domain)
class TestConfig(unittest.TestCase):
"""Test entire configuration.
Currently lower coverage is being obtained since string sets are
being compared rather than returned objects. Comparison logic for
the config objects isn't clear yet and proof that they function is enough.
"""
def setUp(self):
self.config = Config.Config()
domain_policies = self.config._data['acceptable-mxs']
self.mail_domains = ['gmail.com', 'yahoo.com', 'hotmail.com', '123.cn', 'qq.com']
for domain in self.mail_domains:
new = Config.AcceptableMX(domain=domain)
new.add_acceptable_mx('.' + domain)
domain_policies[domain] = new
def testGetAllMxItems(self):
"""Make sure the basic use case of get_all_mx_items functions."""
# [ ('.gmail.com', 'gmail.com'), ('.yahoo.com', 'yahoo.com'), ... ]
control_data = [ ('.' + domain, domain) for domain in self.mail_domains ]
test_data = [ (mx, p.domain) for mx, p in self.config.get_all_mx_items() ]
self.assertListEqual(sorted(test_data), sorted(control_data))
def testGetAllMxItemsMultiMX(self):
config = copy.deepcopy(self.config)
domain_policy = config.acceptable_mxs.get('gmail.com')
# deal with reality, mail.google.com
domain_policy.add_acceptable_mx('.mail.google.com')
control_data = [ ('.' + domain, domain) for domain in self.mail_domains ]
control_data.append(('.mail.google.com', 'gmail.com'))
test_data = [ (mx, p.domain) for mx, p in config.get_all_mx_items() ]
self.assertListEqual(sorted(test_data), sorted(control_data))
def testGetMXtoDomainPolicy(self):
control_data = dict([ ('.' + domain, set([domain]))
for domain in self.mail_domains ])
test_data = {}
for mx, pset in self.config.get_mx_to_domain_policy_map().items():
policy_list = [ p.domain for p in pset ]
test_data[mx] = set(policy_list)
self.assertDictEqual(test_data, control_data)
def testGetMXtoDomainPolicyMultiMX(self):
config = copy.deepcopy(self.config)
domain_policy = config.acceptable_mxs.get('gmail.com')
domain_policy.add_acceptable_mx('.mail.google.com')
control_data = dict([ ('.' + domain, set([domain]))
for domain in self.mail_domains ])
control_data['.mail.google.com'] = set(['gmail.com'])
test_data = {}
for mx, pset in config.get_mx_to_domain_policy_map().items():
policy_list = [ p.domain for p in pset ]
test_data[mx] = set(policy_list)
self.assertDictEqual(test_data, control_data)
if __name__ == '__main__':
unittest.main()

36
bigger_test_config.json Normal file
View file

@ -0,0 +1,36 @@
{
"timestamp": 1401414363,
"author": "Electronic Frontier Foundation https://eff.org",
"expires": "2015-08-01T12:00:00+08:00",
"tls-policies": {
".yahoodns.net": {
"require-valid-certificate": true
},
".eff.org": {
"require-tls": true,
"min-tls-version": "TLSv1.1",
"enforce-mode": "enforce",
"accept-spki-hashes": [
"sha1/5R0zeLx7EWRxqw6HRlgCRxNLHDo=",
"sha1/YlrkMlC6C4SJRZSVyRvnvoJ+8eM="
]
},
".google.com": {
"require-valid-certificate": true,
"min-tls-version": "TLSv1.1",
"enforce-mode": "log-only",
"error-notification": "https://google.com/post/reports/here"
}
},
"acceptable-mxs": {
"yahoo.com": {
"accept-mx-domains": [".yahoodns.net"]
},
"gmail.com": {
"accept-mx-domains": [".google.com"]
},
"eff.org": {
"accept-mx-domains": [".eff.org"]
}
}
}

52
golden-domains.txt
View file

@ -1,22 +1,32 @@
ymail.com
yandex.ru
yahoo.co.uk
wp.pl
web.de
vtext.com
ukr.net
t-online.de
sompo-japan.co.jp
sbcglobal.net
salesforce.com
rogers.com
rocketmail.com
rambler.ru
marktplaats.nl
interia.pl
gmx.net
gmx.de
facebook.com
craigslist.org
bigpond.com
163.com
aol.com
bigpond.com
comcast.net
craigslist.org
facebook.com
gmail.com
gmx.de
hotmail.com
icloud.com
live.com
mac.com
me.com
msn.com
naver.com
outlook.com
qq.com
rocketmail.com
rogers.com
salesforce.com
sbcglobal.net
shaw.ca
sympatico.ca
t-online.de
ukr.net
vtext.com
web.de
wp.pl
yahoo.com
yahoogroups.com
yandex.ru
ymail.com

6734
google-starttls-domains.csv Normal file
View file

File diff suppressed because it is too large Load diff

4
requirements.txt Normal file
View file

@ -0,0 +1,4 @@
dnspython
publicsuffix
m2crypto
dateutils

241
starttls-everywhere.json
View file

@ -1,99 +1,78 @@
{
"tls-policies": {
".valid-example-recipient.com": {
"force-tls" : true
},
".mx.aol.com": {
"min-tls-version": "TLSv1",
"require-tls": true
},
".psmtp.com": {
"min-tls-version": "TLSv1",
"require-tls": true
},
".ukr.net": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".interia.pl": {
"min-tls-version": "TLSv1",
"require-tls": true
},
".gmx.net": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".web.de": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".marktplaats.nl": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".wp.pl": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".yahoodns.net": {
"min-tls-version": "TLSv1",
"require-tls": true
},
".t-online.de": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".rambler.ru": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".t.facebook.com": {
"min-tls-version": "TLSv1",
"require-tls": true
}
},
"acceptable-mxs": {
"valid-example-recipient.com": {
"accept-mx-domains": [ ".valid-example-recipient.com" ]
},
"wp.pl": {
"163.com": {
"accept-mx-domains": [
".wp.pl"
]
},
"yahoo.co.uk": {
"accept-mx-domains": [
".yahoodns.net"
]
},
"rocketmail.com": {
"accept-mx-domains": [
".yahoodns.net"
]
},
"web.de": {
"accept-mx-domains": [
".web.de"
]
},
"sbcglobal.net": {
"accept-mx-domains": [
".yahoodns.net"
".163.com"
]
},
"aol.com": {
"accept-mx-domains": [
".mx.aol.com"
".aol.com"
]
},
"facebook.com": {
"craigslist.org": {
"accept-mx-domains": [
".t.facebook.com"
".craigslist.org"
]
},
"sompo-japan.co.jp": {
"gmail.com": {
"accept-mx-domains": [
".psmtp.com"
".google.com"
]
},
"hotmail.com": {
"accept-mx-domains": [
".outlook.com"
]
},
"icloud.com": {
"accept-mx-domains": [
".icloud.com"
]
},
"live.com": {
"accept-mx-domains": [
".outlook.com"
]
},
"mac.com": {
"accept-mx-domains": [
".icloud.com"
]
},
"me.com": {
"accept-mx-domains": [
".icloud.com"
]
},
"msn.com": {
"accept-mx-domains": [
".outlook.com"
]
},
"naver.com": {
"accept-mx-domains": [
".naver.com"
]
},
"outlook.com": {
"accept-mx-domains": [
".outlook.com"
]
},
"qq.com": {
"accept-mx-domains": [
".qq.com"
]
},
"rocketmail.com": {
"accept-mx-domains": [
".yahoo.com"
]
},
"rogers.com": {
"accept-mx-domains": [
".yahoo.com"
]
},
"salesforce.com": {
@ -101,9 +80,19 @@
".psmtp.com"
]
},
"rambler.ru": {
"sbcglobal.net": {
"accept-mx-domains": [
".rambler.ru"
".yahoo.com"
]
},
"shaw.ca": {
"accept-mx-domains": [
".shaw.ca"
]
},
"sympatico.ca": {
"accept-mx-domains": [
".outlook.com"
]
},
"t-online.de": {
@ -111,40 +100,88 @@
".t-online.de"
]
},
"gmx.net": {
"wp.pl": {
"accept-mx-domains": [
".gmx.net"
".wp.pl"
]
},
"gmx.de": {
"yahoo.com": {
"accept-mx-domains": [
".gmx.net"
".yahoo.com"
]
},
"ukr.net": {
"yahoogroups.com": {
"accept-mx-domains": [
".ukr.net"
".yahoo.com"
]
},
"rogers.com": {
"yandex.ru": {
"accept-mx-domains": [
".yahoodns.net"
".yandex.ru"
]
},
"ymail.com": {
"accept-mx-domains": [
".yahoodns.net"
".yahoo.com"
]
}
},
"tls-policies": {
".163.com": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
"marktplaats.nl": {
"accept-mx-domains": [
".marktplaats.nl"
]
".aol.com": {
"min-tls-version": "TLSv1",
"require-tls": true
},
"interia.pl": {
"accept-mx-domains": [
".interia.pl"
]
".craigslist.org": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".google.com": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".icloud.com": {
"min-tls-version": "TLSv1",
"require-tls": true
},
".naver.com": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".outlook.com": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".psmtp.com": {
"min-tls-version": "TLSv1",
"require-tls": true
},
".qq.com": {
"min-tls-version": "TLSv1",
"require-tls": true
},
".shaw.ca": {
"min-tls-version": "TLSv1",
"require-tls": true
},
".t-online.de": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".wp.pl": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".yahoo.com": {
"min-tls-version": "TLSv1.1",
"require-tls": true
},
".yandex.ru": {
"min-tls-version": "TLSv1.1",
"require-tls": true
}
}
}

7
vagrant-shared/postfix-config-sender.cf
View file

@ -37,10 +37,3 @@ mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
mailbox_size_limit = 0
recipient_delimiter = +
inet_interfaces = all
#STARTTLS EVERYWHERE MAGIC STARTS HERE
smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy
smtp_tls_loglevel = 1
smtp_tls_security_level = may
smtp_tls_CAfile = /etc/certificates/ca.crt